{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ch 9 Multi-Agent Reinforcement Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def init_grid(size=(10,)):\n",
    "    grid = torch.randn(*size)\n",
    "    grid[grid > 0] = 1\n",
    "    grid[grid <= 0] = 0\n",
    "    grid = grid.byte() #A\n",
    "    return grid\n",
    "\n",
    "def get_reward(s,a): #B\n",
    "    r = -1\n",
    "    for i in s:\n",
    "        if i == a:\n",
    "            r += 0.9\n",
    "    r *= 2.\n",
    "    return r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_params(N,size): #A\n",
    "    ret = []\n",
    "    for i in range(N):\n",
    "        vec = torch.randn(size) / 10.\n",
    "        vec.requires_grad = True\n",
    "        ret.append(vec)\n",
    "    return ret\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qfunc(s,theta,layers=[(4,20),(20,2)],afn=torch.tanh):\n",
    "    l1n = layers[0] \n",
    "    l1s = np.prod(l1n) #A\n",
    "    theta_1 = theta[0:l1s].reshape(l1n) #B\n",
    "    l2n = layers[1]\n",
    "    l2s = np.prod(l2n)\n",
    "    theta_2 = theta[l1s:l2s+l1s].reshape(l2n)\n",
    "    bias = torch.ones((1,theta_1.shape[1]))\n",
    "    l1 = s @ theta_1 + bias #C\n",
    "    l1 = torch.nn.functional.elu(l1)\n",
    "    l2 = afn(l1 @ theta_2) #D\n",
    "    return l2.flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_substate(b): #A\n",
    "    s = torch.zeros(2) \n",
    "    if b > 0: #B\n",
    "        s[1] = 1\n",
    "    else:\n",
    "        s[0] = 1\n",
    "    return s\n",
    "\n",
    "def joint_state(s): #C\n",
    "    s1_ = get_substate(s[0]) #D\n",
    "    s2_ = get_substate(s[1])\n",
    "    ret = (s1_.reshape(2,1) @ s2_.reshape(1,2)).flatten() #E\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0],\n",
      "       dtype=torch.uint8)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x125683fd0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAA5CAYAAAAfmaNLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACSRJREFUeJzt3X+MHGUdx/H3x7aU8PtKCZRKxPqDiFGhXAAVSWP50RLTqkEDMVoEgkQbxcQoCUlD+McWoyYaoqlAREKgEQWqKYEiEP8wVAq5FsqvFoKBWlootYUQ0eLXP+Y5btjO3u3d7O3us/d5JZOd2Xl25/vNM7Pf25nZ5xQRmJmZWT7e1+0AzMzMbHxcvM3MzDLj4m1mZpYZF28zM7PMuHibmZllxsXbzMwsM7WKt6RZktZL2poeB5q0e0fSUJrW1tmmmZnZVKc6v/OWdD3wekSslHQ1MBARP6po92ZEHFYjTjMzM0vqFu9ngQURsUPSHODhiDipop2Lt5mZWZvULd7/ioij0ryAPcPLDe32A0PAfmBlRNzd5P2uAK4AmMa00w7hiAnH1sxHP/lW299z2HObD5mU952smCcr3hx5vxgxmftFbjFP5n5hI/xZNOIN9rwWEceM1W7M4i3pAeC4ilXXALeUi7WkPRFxwHVvSXMjYrukecCDwMKIeH607R6hWXGGFo4V/7jd98+htr/nsPOPP2VS3neyYp6seHPk/WLEZO4XucU8mfuFjfBn0YgH4s7HImJwrHbTx2oQEec0Wydpp6Q5pdPmu5q8x/b0+IKkh4FTgVGLt5mZmVWr+1OxtcCyNL8MuKexgaQBSTPT/Gzgs8BTNbdrZmY2ZdUt3iuBcyVtBc5Jy0galHRjavMxYKOkTcBDFNe8XbzNzMwmaMzT5qOJiN3AARemI2IjcHma/xvwiTrbMTMzsxEeYc3MzCwzLt5mZmaZaUvxlrRI0rOStqWR1hrXz5S0Jq3fIOnEdmzXzMxsKqpdvCVNA24AFgMnAxdLOrmh2WUUA7h8GPg5sKruds3MzKaqdnzzPh3YFhEvRMR/gDuApQ1tlgK3pPk7gYVpRDYzMzMbp3YU77nAS6Xll9NzlW0iYj+wFzi6Dds2MzObcmr9VKzdymObH4zHujUzM6vSjm/e24ETSsvvT89VtpE0HTgS2N34RhGxOiIGI2JwBjPbEJqZmVn/aUfxfhT4iKQPSjoIuIhi2NSy8jCqFwIPRp1/Z2ZmZjaF1T5tHhH7JS0H7gOmATdHxBZJ1wEbI2ItcBNwq6RtwOsUBd7MzMwmoC3XvCNiHbCu4bkVpfl/A19px7bMzMymOo+wZmZmlhkXbzMzs8y4eJuZmWWmU2ObXyLpVUlDabq8Hds1MzObimrfsFYa2/xcitHVHpW0NiKeami6JiKW192emZnZVNepsc3NzMysTVR3rBRJFwKLIuLytPx14Izyt2xJlwA/Bl4FngO+HxEvVbzXu8OjAicBz44jlNnAaxPJIQP9nBs4v9w5v3z1c26QZ34fiIhjxmrUqbHN/wTcHhFvS/oWxX8Y+3xjo4hYDayeyAYkbYyIwXph9qZ+zg2cX+6cX776OTfo7/w6MrZ5ROyOiLfT4o3AaW3YrpmZ2ZTUkbHNJc0pLS4Bnm7Dds3MzKakTo1t/l1JS4D9FGObX1J3uxUmdLo9E/2cGzi/3Dm/fPVzbtDH+dW+Yc3MzMw6yyOsmZmZZcbF28zMLDNZFe8WhmGdKWlNWr9B0omdj3JiJJ0g6SFJT0naIul7FW0WSNpbGmZ2RdV79SpJL0p6IsW+sWK9JP0i9d9mSfO7EedESDqp1C9DkvZJuqqhTVb9J+lmSbskPVl6bpak9ZK2pseBJq9dltpslbSsc1G3rkl+P5H0TNr/7pJ0VJPXjrovd1uT3K6VtL20/13Q5LWjfs72gib5rSnl9qKkoSav7em+a1lEZDFR3Az3PDAPOAjYBJzc0ObbwK/T/EUUQ7J2PfYW85sDzE/zh1MMZtOY3wLgz92OtUaOLwKzR1l/AXAvIOBMYEO3Y55gntOAVygGW8i2/4CzgfnAk6XnrgeuTvNXA6sqXjcLeCE9DqT5gW7n02J+5wHT0/yqqvzSulH35W5PTXK7FvjBGK8b83O2F6aq/BrW/xRYkWPftTrl9M27lWFYl1IMAANwJ7BQkjoY44RFxI6IeDzNv0Hxc7q53Y2q45YCv4vCI8BRDT8zzMVC4PmI+Ee3A6kjIv5K8euQsvIxdgvwxYqXng+sj4jXI2IPsB5YNGmBTlBVfhFxf0TsT4uPUIxbkZ0mfdeKLIa7Hi2/9Jn/VeD2jgbVYTkV77lAeUjVlzmwuL3bJh2Ae4GjOxJdG6XT/acCGypWf1rSJkn3Svp4RwOrL4D7JT2WhsJt1Eof5+Aimn9w5Nx/AMdGxI40/wpwbEWbfunHSynOBFUZa1/uVcvTJYGbm1zy6Ie++xywMyK2Nlmfa9+9R07Fe0qQdBjwB+CqiNjXsPpxilOxnwJ+Cdzd6fhqOisi5gOLge9IOrvbAbVbGqhoCfD7itW59997RHEOsi9/ayrpGopxKW5r0iTHfflXwIeAU4AdFKeW+9HFjP6tO8e+O0BOxXvMYVjLbSRNB44EdnckujaQNIOicN8WEX9sXB8R+yLizTS/DpghaXaHw5ywiNieHncBd1GcoitrpY973WLg8YjY2bgi9/5Ldg5fykiPuyraZN2PKv6R0heAr6U/UA7Qwr7ccyJiZ0S8ExH/A35Ddcy599104MvAmmZtcuy7KjkV7zGHYU3Lw3e2Xgg82Ozg6zXpOs1NwNMR8bMmbY4bvoYv6XSK/svijxNJh0o6fHie4sagJxuarQW+ke46PxPYWzpFm4umf/Xn3H8l5WNsGXBPRZv7gPMkDaRTs+el53qepEXAD4ElEfFWkzat7Ms9p+H+kS9RHXMrn7O97BzgmYh4uWplrn1Xqdt3zI1norgb+TmKuyGvSc9dR3GgARxMcbpyG/B3YF63Yx5HbmdRnILcDAyl6QLgSuDK1GY5sIXiDtBHgM90O+5x5Dcvxb0p5TDcf+X8BNyQ+vcJYLDbcY8zx0MpivGRpeey7T+KP0J2AP+luPZ5GcU9JH8BtgIPALNS20HgxtJrL03H4Tbgm93OZRz5baO45jt8DA7/euV4YF2ar9yXe2lqktut6bjaTFGQ5zTmlpYP+Jzttakqv/T8b4ePt1LbrPqu1cnDo5qZmWUmp9PmZmZmhou3mZlZdly8zczMMuPibWZmlhkXbzMzs8y4eJuZmWXGxdvMzCwz/wekU9gc6gHF4gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,5))\n",
    "size = (20,) #A\n",
    "hid_layer = 20 #B\n",
    "params = gen_params(size[0],4*hid_layer+hid_layer*2) #C\n",
    "grid = init_grid(size=size)\n",
    "grid_ = grid.clone() #D\n",
    "print(grid)\n",
    "plt.imshow(np.expand_dims(grid,0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 200\n",
    "lr = 0.001 #A\n",
    "losses = [[] for i in range(size[0])] #B\n",
    "for i in range(epochs):\n",
    "    for j in range(size[0]): #C\n",
    "        l = j - 1 if j - 1 >= 0 else size[0]-1 #D\n",
    "        r = j + 1 if j + 1 < size[0] else 0 #E\n",
    "        state_ = grid[[l,r]] #F\n",
    "        state = joint_state(state_) #G\n",
    "        qvals = qfunc(state.float().detach(),params[j],layers=[(4,hid_layer),(hid_layer,2)])\n",
    "        qmax = torch.argmax(qvals,dim=0).detach().item() #H\n",
    "        action = int(qmax)\n",
    "        grid_[j] = action #I\n",
    "        reward = get_reward(state_.detach(),action)\n",
    "        with torch.no_grad(): #J\n",
    "            target = qvals.clone()\n",
    "            target[action] = reward\n",
    "        loss = torch.sum(torch.pow(qvals - target,2))\n",
    "        losses[j].append(loss.detach().numpy())\n",
    "        loss.backward()\n",
    "        with torch.no_grad(): #K\n",
    "            params[j] = params[j] - lr * params[j].grad\n",
    "        params[j].requires_grad = True\n",
    "    with torch.no_grad(): #L\n",
    "        grid.data = grid_.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualization of 1D Ising Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "       dtype=torch.uint8) tensor(1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1258ce5f8>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAADTCAYAAABp7RNGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt4VfWd7/H3NxcSSCBcBbwiKCo9cgSp4gXH1rst2rEzlnaOWrXH6XPq0+pM6+j0HA+nz/QZbWt77LGnHaZ61I4ttg4qVlu1SlusgoIgKBcJiB1jgHBPIAm5fM8fawV3wt7JTvbea+/s9Xk9T56srPVb+/fltxff31q/dTN3R0RE4qUk3wGIiEj0lPxFRGJIyV9EJIaU/EVEYkjJX0QkhpT8RURiSMlfRCSGlPxFRGJIyV9EJIbK8h1AKmPHjvVJkyblOwwRkUFl5cqVO919XF/lCjb5T5o0iRUrVuQ7DBGRQcXM3k+nnIZ9RERiSMlfRCSGCnbYJxvWL13C0oWP0rhrJ8PHjGXOvOs5bc4n8h2WiEjeFW3yX790CS8seID2Q60ANO5s4IUFDwCoAxCR2CvaYZ+lCx89nPi7tB9qZenCR/MUkYhI4Sja5N+4a2e/5ouIxEnRJv/hY8b2a76ISJwUbfKfM+96yoZUdJtXNqSCOfOuz1NEIiKFo2hP+Had1NXVPiIiRyra5A9BB6BkLyJypKId9hERkdSU/EVEYkjJX0QkhpT8RURiSMlfRCSGlPxFRGJIyV9EJIaU/EVEYkjJX0QkhpT8RURiSMlfRCSGlPxFRGJIyV9EJIaU/EVEYkjJX0Qkhor6ef4AT62q47vPb+TDvc0cPXIo37jsFD4z45h8hyUikldFnfyfWlXHXYvW0tzWAUDd3mbuWrQWQB2AiMRaUQ/7fPf5jYcTf5fmtg6++/zGPEUkIlIYijr5f7i3Oen8ur3NPLWqLuJoREQKR1En/6NHDk257K5Fa9UBiEhsFXXy/8ZlpzC0vDTpMg3/iEicFfUJ366Turc9vjrp8lTDQiIixa6ok/+zW57l/26+n+Gn1tPZNpLWhsto3z/j8PLehoVERIpZ0Sb/Z7c8y92v/A8OeRsYlAzZS+XERbQA7ftnMLS8lG9cdkq+wxQRyYuiHfP/3mv3Bok/gZW0UTHueUZVwz9fc7qu9ReR2Cra5L+zbU/S+SXlexl36v1K/CISa0Wb/Ktakl/lU9VcyrYD2yKORkSksESa/M1sq5mtNbPVZrYil3WdV3c8pe3WbV5pu/Gx94ZjZjy75dlcVi8iUtDyccL3E+6+M9eVfOnS22l5+p8YOn4Sn983l3Hto9nve1k07Dle9z8x/9X5AHxq8qdyHYqISMEp2qt9TpvzCW6rM8pXdVJm5QCMtFF8ofWvaNjXyu9rVnD/m/cr+YtILEWd/B14wcwc+Bd3X5DLyoZuBsLE36XSK/j6hzcA8AdW5rJ6EZGCFXXyP9/d68zsKOBFM9vg7n/sWmhmtwC3ABx//PEZVVS/7Wm8cRSWZFkppXzjwxs589B/yqgOEZHBKtITvu5eF/7eATwJnNVj+QJ3n+Xus8aNG5dRXVs2f4/2yl0pl5dgXLTz4xxYtSOjekREBqPI9vzNrAoocffGcPpS4Fu5qq+ltZ6Gk55gwrobKemsSB4Txp5fBg93q5pxVK5C6Zf1S5ewdOGjNO7ayfAxY5kz73pOm/OJfIclIkUmymGf8cCTZtZV78/d/be5qqyyYiKNRy8DYOLb/xUj+XX/OOxdtAnIfwewfukSXljwAO2HWgFo3NnAcw/cx3MP3MfwsePUEYhI1pi75zuGpGbNmuUrVgz8VoD6bU+zYe3tdJYawz+czcS3/xZLegYgYENLOeZ/njvg+rJhwVdupHFnQ5/lKqqHYwYtTU06OhCRbsxspbvP6qtc0d7hO3HC1UzeWkplSweNE1+jccKLBBcbJefNHXkf/2/cld7tD61NjbQ0NoL74aOD+z73aRZ85UbWL12S4yhFpBgUbfIH+H/bv8CM5Qe4aOkupu39IaPKvgd0pCy/e+E6Gh7M2UhUn4aPGZvR+uoIRCRdRTvsA3Dinc8yt+QV7ij7JUfbLj70MTxz6Gt8mjNSDgF5+yEqT2tj3M2XZ1T3QPQc888WDROJxEe6wz5FnfzPu+dl6pK8res5G84ITz3+33moieO/f0VGdQ/U4at90hj7z5Q6BZHio+QPPLWqjrsWraW57aOhnqHlpfzLmZOYsnIX3taZdD13p3Lqwbzs/SeKsiNIpE5BZPBS8g89tPgPbHzzVSq9lRar4JSZ53LTVX/BgVU72L1wHWbJLwF172D0vGl5v/yzS+L1/xVV1UFybmyMNIbETuFwDOogRAqKkj+wZs0annnmGdraPnqjV3l5OXPnzmX69Ok0PPhbWt4dRnjvwZFKnFF/fWrBdADJ5OvoIBV1ECL5peQP/OAHP2Dfvn1HzK+pqeH2228H4P2vv0hpWWXqDynt5Nhv/0VGcUSl0DqCVFJ1EKmm1XGIpE/JH5g/f37KZddccw2VzUex4efrmV5ZSlmKvX93Z/S8wt77T6YQholyob8dhzoUiZvYJ/93l29j4TMP0Vl2KOny8vJyappPwXaO5phyY+awUkpSDf/gjPrc4OsAkinWTiET2epQ1BlJIYh98n/kH//EzoN/pnHEJihJflUPHaWMazgPgGPKjTOHlaYc/7fyEkZec3JRdADJqFPIv7IhFVx6y63qACQj6Sb/on2TV9PuVioZD0BjzUaS3tNV0kFL5XYqW8ZT1+ac7lCRYuff2zrZ//zWok3+p835RNKkk7RTSNxrVQeRNe2HWlm68FElf4lE0Sb/6uoOmppKqWwZz4HqrXSWJblr1uBA9VYqW4JOYm1zB2cMSz3+37G3JZchF6RUnUKi9UuX8NLDC2htUkeQqUI/WS/Fo2iT/znV/8aSps/RTiVVTZNS7v13lrZ22/vnYEfK8X93OLBqR9Hu/Sfqc49fe/85s37pEu39S84VbfKfytMwYhevNf0XaD6KA8Nr6SxN8lA3IzgvAN06gGTj/2bGnsc3APl/9n8m+juUk7hHn2paskdDPxKFok3+1BzLVJYyddhS9m0dymvLp/PGWWfRUZbkn1zSSePw2sPDP3VtzpkpP9jY86vC7QCyldglf9J9tLdIJoo3+V90NzzzVWhrpmZSM5Pf3AqvG8vOmQ3JxvQTTv4CNHfCsBQv/6LT2PPE+gEl/2y9pjGdq3OU2AenTB/tLZKO4k3+068Nfr/0LXzfB4yd2UjnG1tZc3A6B6uqjixv4VVBBMM/61p6P/nr7dbv8f+BvKaxv0k+DgrhuvwBT/dxfqRsSAVz5l0fUUtKnBVv8gfqj6pgy1mjaG5ppvnjw5g6aS/TX3uLZbPPSb73nzD+X9cyvteTv2b9f/n70oWPpnxWf+POBl5Y8EC3sj2v/BgsST4XybmYboJKNTRXTP9GKXxFe5NX/ban2bDuH+jko4e60emc8Eorv2y5jkOVvTzPpx83f4EzbPZERn/m5D5jum/e3OCSoUEmnWSuxCVSGGJ/k9eWDd/qnvgBSoz3zx/Cmd9fyeszz05+8hf6dfMXGAeX1VNxQk2fRwDDx4wtuOu49RROkXgq2uTf0rE3xV29JUyevQ5/1Vh+1my8JMlrjHuM//d18xekNwQ0Z971KV/TeGjEaA6NOwYvH4K1HWJIQx1D9u/u65/ZKyV2EUmlaJN/ZWsHLZXJL9f583kVnOTvULKik1dnnpfx+D8ADnse30jr+/tSDgF1Jdue4/mHRoymdeIJUBLE60MqaD36RFqPPjGtjkBv3hKR/iraMf/f/OxUyo9uS5rXAXBnQn0z/7bli1R0VqT+oH6N/wdKhpVRM3dKn8NAXSf+6msm4EN6iSH8jko62qna1wAN9UryIpJU7J/qeddds7h4zh7o5bwuQHN7CVtqz2b3jpOSF3CwzjKqG6dQ2TKe0ytLOLGipM8OoD9PAe3tvQOpDB06FIDm5mZqamq46KKLmD59er8/R0SKS+xP+F7x0gFeL7+Ahy/8PDsZSzWNgNFENWPZybU8xnm8wtCyTqad8hp1w3fy3ubZR36QgZe201izkUPl+1jbOJXdHd7H8/+Dp4DuXVybVvKvqalJ+sax3jQ3Nx+e3rdvH4sWLWLRokXqCEQkLUW75//D627mO9d9mfby8uQFvJOL+S038mDwp0NbWwVbNs+ioWFyinU+OgqY0jEhjSEgB6zPYaBk7xrOBh0diMRP7Id9pv76FfZXVfdeyJ1qGrmeBzmPV7pm8eGHJ7Ml2VHA4fWg4uAEzms7Na0hoC69dQJr1qzhpZde6vcRwEAkdgrqIESKS+yT/4SXVyW/iieZsA3G0sC1PMa5/grtbRVs7uMoAKDUy5jdNpVTOyek3QkAlI6sYMRlk/LeEaSSqoNQZyFS2JT8l6we2IoJw0FdTdPaWsXW987otSOooIxz2qZyUufEfldZ6B1BOtLpLNRxiORe7JP/SU+/QVNVKZQmuYmrL2GbGI5jjKWBv/bgBHGvHUHYlNVeyaz2yTntCBIT6WDX345DnYtIarFP/jO+/SINVaW0Tx0BlaXQ4VBq6Q8F9ZTQTtU0cp0H5wlSDg8lNGv/jwqCE8WQ3j0Dg+XoIF9y0bnkelqdlwxU7JP/iXc+S89/WfuEobSfVgPlJQPvBLocbrePEnWvnUKSZk63U/AkdXVNe+te2rb8hkOb/kDZxInsuv46Xtuzp+iODuKsEDojTQ+eTj/2yf/0f3yKxs4jL/McXtLG2AsmsX6IZ94BJNNLooaPhpLG0MC1iUNJW5IPJaXTQXT/DpPX29HezPLS9awfuofy1lYMOFRR0W06CDAHbSIiaSsvL2fu3LkD7gBin/yv++8/5NX2SXTw0fN9Sung3LKt/OyfvspdyzbzyJ69dFaE5wTykfTSSNqppruOMma1rkq740j7u7bktTbRwio2sJUP1VmI5FBNTQ233377gNaN/R2+M8d0wK6tvNl+LAcYQhWHmFn2AVPKdjN//nxGDh3Kb6+4gi0dY7jjtVr2T6nOznBQf3Srq3/TTYzgx9wWPL5iWurO4v5+dCh9Tw8HxnLEUYzvYk7dciZsPvImtQqCS2FP6phwxLK+OB81UarO6I2yzWwp297vzxYpZFGcvyvaPf81a9bw1FNPUbKngYoddVj7oT7XWXfSdF4670paKxNe86g92/RkcBSj6cE1XdPm/P36Fq6obyd7POE+meT1erfp4t4ZGNbSwh333DOgdWM/7APwnTv+npI/b8K8c0DrrztpOkvPvoT91SOD5Na1capDkLjr49yWpgc2XU0jV7+3lEt++Ucuf/55BiL2wz4Ak/YMYcfHv8ZPTxvF9kpj5IEDnLvsOabVrklr/Wm1a44o23V00HL46CDhS1SnIHFxeFsf+NClpo+cbmIEj594KVwLl5NbRZv8dz+1id3TPsX3Tq+ipTRo3D3V1bxw4TUAaXcAPSXrEKCPTqHbNOokRCSldivnt5PO5r4c1xNp8jezy4H7gVLgp+4+sEGtNDQt28ZPLvgo8XdpKytj6dmXDDj5p5KqU+gp5VBSfw8V1YGIFK1dNjrndUSW/M2sFPgRcAnwAfCGmS1293U5qc+d7ZXJE+T+6pG5qDIt6XYSvVl30nRevuBKmsv7OsrI9TTqhERyYIxn9v7udES5538WUOvuWwDMbCFwNZCT5F9rH1DVOommymFHLBvRtDfpOg7hxYvQfUyOFHPyIxsdSDZsPe8kXv7Yp9hlY8I5hXPiTNM5nFaHn1Nl3sblW5cDF+e2npx+enfHAP+R8PcHwNm5quwNW8/ZWw7wh1Nm0F760T+zrKOdT75dTuWovztincYhu3nszP8FDp9977PdlpXt20VFQ3qXjMbFpD/VctOf7s93GLGWy2tuRk7ZxzFn76C8uv3wCOWfOJ9HuYkDjMhhzfGdDq72eYVb1s8g1wrqhK+Z3QLcAnD88cdn9FkHh5RwckMdAMsnf4ymiqFUtzZz9pZ3OP3PU5KuU31oFABHtY3mS60XdV9YCRwHtSX1vFb2Lq2W4hrnfOwUhRmgwsvAoJX2lNOZPHFUYqYTeK37rFOBmwFoijyceDBgDl7qfZbMVJTJvw44LuHvY8N5h7n7AmABBNf5Z1LZsNZWDlZWcnJD3eFOAKCkvQJInvybhuyhorOcG3bMxd3DIaAgm3dNT+mYwOSO8YfnJ77ApbaknhVlW2iylvQDTdZZKJmLxNrBjtx3rlEm/zeAk83sRIKkPw/4Qq4qu+DUU3lh8xY6E5/n31lCVdOkpOU7rJ23Ji7hnA0X0PDnlfyy8/d91nF81WnMGH0xFaXBU/kmd4xnSvgYg2QdR8/p2tKgszhgwcPVHKjyCma1T+nX4xD68wYxESlsHd7O/uP6LpepSO/wNbMrgf9NcKnnQ+7+7VRls3GH77KHH2bJxlpah5RR0lFBVeMJVLaOT4wIgMqqMuZcO5WpZ6eXcN9dvo3Xnt5M0+7WjOLLhmPKjdOHljJE+X/QS2eHQdPFPd3a2cKeY9v5+NeuZKD0eAcRkRhKN/kP4B2HIiIy2Cn5i4jEUMEO+5hZA/B+lj5uLLAzS5+VLYUYExRmXIUYExRmXIUYExRmXIUYE2Qe1wnuPq6vQgWb/LPJzFakMwYWpUKMCQozrkKMCQozrkKMCQozrkKMCaKLS8M+IiIxpOQvIhJDcUn+C/IdQBKFGBMUZlyFGBMUZlyFGBMUZlyFGBNEFFcsxvxFRKS7uOz5i4hIAiV/EZEYKurkb2aXm9lGM6s1szvzGMdxZrbEzNaZ2Ttm9rVw/nwzqzOz1eHPwB/oMbC4tprZ2rDuFeG80Wb2opltCn+PijimUxLaY7WZ7Tez26JuKzN7yMx2mNnbCfOSto0FfhhuZ2vMbGbEcX3XzDaEdT9pZiPD+ZPMrDmhzX4SYUwpvy8zuytsq41mdlkuYuolrscTYtpqZqvD+VG1VapcEP225e5F+UPw8LjNwGRgCPAWMC1PsUwEZobTw4F3gWnAfODreWyjrcDYHvO+A9wZTt8J3Jvn73AbcELUbQVcAMwE3u6rbYArgd8QPClwNrA84rguBcrC6XsT4pqUWC7imJJ+X+F2/xZQAZwY/h8tjSquHsvvA+6OuK1S5YLIt61i3vM//NpIdz8EdL02MnLuXu/ub4bTjcB6gjebFaKrgUfC6UeAz+QxlouAze6erTu90+bufwR6vkg1VdtcDTzqgWXASDPLyQsWksXl7i+4e9fbhZYRvCsjMinaKpWrgYXu3uru7wG1BP9XI43LguegXwv8Ihd19xJTqlwQ+bZVzMk/2Wsj855wzWwSMANYHs66NTyceyjqIRaCVwi8YGYrLXiLGsB4d68Pp7cB45OvGol5dP/Pmc+2gtRtU0jb2k0Ee4pdTjSzVWb2BzObE3Esyb6vQmmrOcB2d9+UMC/StuqRCyLftoo5+RccM6sG/h24zd33Az8meK3YGUA9wWFolM5395nAFcBXzOyCxIUeHHfm5VpgMxsCXAX8KpyV77bqJp9tk4qZfRNoBx4LZ9UDx7v7DODvgJ+b2YhU62dZQX1fSXye7jsWkbZVklxwWFTbVjEn/z5fGxklMysn+LIfc/dFAO6+3d073L0T+FdydPibirvXhb93AE+G9W/vOqwMf++IMqYEVwBvuvv2MMa8tlUoVdvkfVszsy8Cnwb+JkwehEMru8LplQTj61OjiKeX76sQ2qoMuAZ4vGtelG2VLBeQh22rmJP/4ddGhnuR84DF+QgkHF98EFjv7t9PmJ84dveXwNs9181hTFVmNrxrmuCk4dsEbXRDWOwG4OmoYuqh255ZPtsqQaq2WQxcH16ZMRvYl3AIn3NmdjlwB3CVux9MmD/OzErD6cnAycCWiGJK9X0tBuaZWYUFr3Q9GXg9ipgSXAxscPcPumZE1VapcgH52LZyfXY7nz8EZ8rfJejFv5nHOM4nOIxbA6wOf64EfgasDecvBiZGGNNkgqsu3gLe6WofYAzwErAJ+B0wOg/tVQXsAmoS5kXaVgQdTz3QRjDOenOqtiG4EuNH4Xa2FpgVcVy1BOPCXdvWT8Kynw2/29XAm8DcCGNK+X0B3wzbaiNwRZRtFc5/GPhyj7JRtVWqXBD5tqXHO4iIxFAxD/uIiEgKSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMaTkLyISQ0r+IiIxpOQvIhJDSv4iIjGk5C8iEkNK/iIiMZRR8jez0Wb2opltCn+PSlGuw8xWhz+LM6lTREQyZ+4+8JXNvgPsdvd7zOxOYJS7/0OSck3uXp1BnCIikkWZJv+NwIXuXm9mE4Hfu/spScop+YuIFJBMx/zHu3t9OL0NGJ+iXKWZrTCzZWb2mQzrFBGRDJX1VcDMfgdMSLLom4l/uLubWarDiBPcvc7MJgMvm9lad9+cpK5bgFsASik9cxgj+vwHSHqmTj+Y0frvrhmWpUhEJJca2bPT3cf1VS6SYZ8e6zwM/Nrdn+it3Agb7WfbRQOOTbp7/sPVGa1/2dFnZCkSEcml3/kTK919Vl/lMh32WQzcEE7fADzds4CZjTKzinB6LHAesC7DekVEJAOZJv97gEvMbBNwcfg3ZjbLzH4aljkNWGFmbwFLgHvcXclfRCSP+hzz74277wKOGJtx9xXAl8LpV4HTM6lHRESyKyt3+JrZ5Wa20cxqw+v9ey6vMLPHw+XLzWxSNuoVEZGByTj5m1kp8CPgCmAa8Hkzm9aj2M3AHnc/CfgBcG+m9YqIyMBlY8//LKDW3be4+yFgIXB1jzJXA4+E008AF5mZZaFuEREZgGwk/2OA/0j4+4NwXtIy7t4O7APG9PwgM7slvBlsRRutWQhNRESSKainerr7Anef5e6zyqnIdzgiIkUrG8m/Djgu4e9jw3lJy5hZGVAD7MpC3SIiMgDZSP5vACeb2YlmNgSYR3DzV6LEm8H+CnjZM7m1WEREMpLRdf4QjOGb2a3A80Ap8JC7v2Nm3wJWuPti4EHgZ2ZWC+wm6CBERCRPMk7+oU7Aw58OAHe/O2H5POBCguGfIcAngS1ZqltERPop4+SfcJ3/JQRX+rxhZouTPMLhcXe/NdP6REQkc1Fd5y8iIgUkquv8AT5rZmvM7AkzOy7JchERiUi2xvz78gzwC3dvNbO/Jbjb95M9CyW+zAVo+p0/sbGPzx0L7MxqpLmR9zhLJ6ZVrJc4a7MXTGby3pZpUpzZpTjTd0I6hTJ6mQuAmZ0DzHf3y8K/7wJw939OUb6U4KXvNRlVHHzWinReWpBvijN7BkOMoDizTXFmXyTX+Ydv+epyFbA+C/WKiMgARXWd/1fN7CqgneA6/y9mWq+IiAxcVsb83f054Lke8+5OmL4LuCsbdfWwIAefmQuKM3sGQ4ygOLNNcWZZxmP+IiIy+BTUUz1FRCQagyL5F/prIs3sODNbYmbrzOwdM/takjIXmtk+M1sd/tyd7LMiiHWrma0NY1iRZLmZ2Q/DtlxjZjPzEOMpCe202sz2m9ltPcrkpT3N7CEz22FmbyfMG21mL5rZpvD3qBTr3hCW2WRmNyQrk+M4v2tmG8Lv9UkzG5li3V63kQjinG9mdQnf7ZUp1u01L0QQ5+MJMW41s9Up1o2sPfvF3Qv6h+Ak8mZgMsFzgd4CpvUo89+An4TT8wgeJRFljBOBmeH0cODdJDFeCPy6ANpzKzC2l+VXAr8BDJgNLC+A738bcEIhtCdwATATeDth3neAO8PpO4F7k6w3muB5VqOBUeH0qIjjvBQoC6fvTRZnOttIBHHOB76exnbRa17IdZw9lt8H3J3v9uzPz2DY8y/410S6e727vxlONxJcyprsLufB4GrgUQ8sA0b2uFQ3ahcBm939/TzGcJi7/5HgirVEidvfI8Bnkqx6GfCiu+929z3Ai8DlUcbp7i948CY9gGUE797IqxTtmY5IHyvTW5xhrrkW+EWu6s+FwZD8s/aayCiEQ04zgOVJFp9jZm+Z2W/M7GORBvYRB14ws5XhHdU9pfu4jqjMI/V/qkJoT4Dx7l4fTm8DxicpU2jtehPBEV4yfW0jUbg1HJ56KMUwWiG15xxgu7tvSrG8ENrzCIMh+Q8aZlYN/Dtwm7vv77H4TYKhi/8M/B/gqajjC53v7jOBK4CvmNkFeYqjT+FNg1cBv0qyuFDasxsPjvML+hI6M/smwT03j6Uoku9t5MfAFOAMoJ5gSKWQfZ7e9/rz3Z5JDYbkPyheE2lm5QSJ/zF3X9Rzubvvd/emcPo5oNzMxkYZY1h3Xfh7B/AkweFzonTaOypXAG+6+/aeCwqlPUPbu4bGwt87kpQpiHY1sy8Cnwb+JuyojpDGNpJT7r7d3TvcvRP41xT1F0p7lgHXAI+nKpPv9kxlMCT/gn9NZDjm9yCw3t2/n6LMhK7zEGZ2FkHbR91BVZnZ8K5pghOAb/cothi4PrzqZzawL2FII2op96gKoT0TJG5/NwBPJynzPHCpmY0KhzEuDedFxswuB+4ArnL3gynKpLON5FSPc0x/maL+dPJCFC4GNrj7B8kWFkJ7ppTvM87p/BBcgfIuwdn9b4bzvkWwEQNUEgwN1AKvA5Mjju98gkP9NcDq8OdK4MvAl8MytwLvEFyVsAw4Nw/tODms/60wlq62TIzTCF7OsxlYC8zK03deRZDMaxLm5b09CTqjeqCNYJz5ZoLzSy8Bm4DfAaPDsrOAnyase1O4jdYCN+YhzlqCcfKubbTrCrmjged620YijvNn4ba3hiChT+wZZ/j3EXkhyjjD+Q93bZMJZfPWnv350R2+IiIxNBiGfUREJMuU/EVEYkjJX0QkhpT8RURiSMlfRCSGlPxFRGJIyV9EJIaU/EVEYuiM4t1pAAAABklEQVT/AxTiHuCSxNF9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(2,1)\n",
    "for i in range(size[0]):\n",
    "    ax[0].scatter(np.arange(len(losses[i])),losses[i])\n",
    "print(grid,grid.sum())\n",
    "ax[1].imshow(np.expand_dims(grid,0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque #A\n",
    "from random import shuffle #B\n",
    "\n",
    "def softmax_policy(qvals,temp=0.9): #C\n",
    "    soft = torch.exp(qvals/temp) / torch.sum(torch.exp(qvals/temp)) #D\n",
    "    action = torch.multinomial(soft,1) #E\n",
    "    return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coords(grid,j): #A\n",
    "    x = int(np.floor(j / grid.shape[0])) #B\n",
    "    y = int(j - x * grid.shape[0]) #C\n",
    "    return x,y\n",
    "\n",
    "def get_reward_2d(action,action_mean): #D\n",
    "    r = (action*(action_mean-action/2)).sum()/action.sum() #E\n",
    "    return torch.tanh(5 * r) #F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.8483) tensor(0.8483)\n"
     ]
    }
   ],
   "source": [
    "x1 = get_reward_2d(torch.Tensor([1,0]),torch.Tensor([0.25, 0.75]))\n",
    "x2 = get_reward_2d(torch.Tensor([0,1]),torch.Tensor([0.25, 0.75]))\n",
    "print(x1,x2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_action(grid,j):\n",
    "    x,y = get_coords(grid,j) #A\n",
    "    action_mean = torch.zeros(2) #B\n",
    "    for i in [-1,0,1]: #C\n",
    "        for k in [-1,0,1]:\n",
    "            if i == k == 0:\n",
    "                continue\n",
    "            x_,y_ = x + i, y + k\n",
    "            x_ = x_ if x_ >= 0 else grid.shape[0] - 1\n",
    "            y_ = y_ if y_ >= 0 else grid.shape[1] - 1\n",
    "            x_ = x_ if x_ <  grid.shape[0] else 0\n",
    "            y_ = y_ if y_ < grid.shape[1] else 0\n",
    "            cur_n = grid[x_,y_]\n",
    "            s = get_substate(cur_n) #D\n",
    "            action_mean += s\n",
    "    action_mean /= action_mean.sum() #E\n",
    "    return action_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(45)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACkBJREFUeJzt3M+LXfUdxvHn6SRWR0u1mo1JaLJQSxBbZQj+ABdGiFbRTRcKCnWTTdUogmg3/gMiuhAh+GOj6CJmISKOBXXRTeoYg5pEg6g1MRHHlKqYRUx8upgpRDFzTzLn65n74f0CwYzH68Mwb8+dO3e+TiIANf1q6AEA2iFwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwpb1uJBz/vdRNasXt774+59d7L3x0RbF15yeOgJJX2673t99Z9jHnVdk8DXrF6uf02v7v1xN57/p94fE21NT+8cekJJ6zfu63QdT9GBwggcKIzAgcIIHCiMwIHCCBworFPgtq+z/aHtj2w/0HoUgH6MDNz2hKTHJV0vaZ2kW22vaz0MwOJ1uYOvl/RRko+THJH0gqSb284C0Icuga+UdPzbZvbPf+xHbG+yPWN7ZvbQsb72AViE3l5kS7IlyVSSqRXnTvT1sAAWoUvgn0s6/o3lq+Y/BmCJ6xL4W5IusL3W9mmSbpH0UttZAPow8rfJkhy1faekaUkTkp5Osqv5MgCL1unXRZO8IumVxlsA9Ix3sgGFEThQGIEDhRE4UBiBA4U1OXQR0vSBNocNtjp4stXeVsbpAM4hP7fcwYHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwpqcqrr33cmxOvUSaKlFC3tzqNN13MGBwggcKIzAgcIIHCiMwIHCCBwobGTgtlfbfsP2btu7bG/+JYYBWLwuPwc/Kum+JDts/0bS27b/kWR3420AFmnkHTzJwSQ75v/+W0l7JK1sPQzA4p3U9+C210i6VNL2FmMA9KvzW1VtnyXpRUn3JPnmZ/75JkmbJOl0TfY2EMCp63QHt71cc3E/l2Tbz12TZEuSqSRTy/XrPjcCOEVdXkW3pKck7UnySPtJAPrS5Q5+laTbJV1je+f8X39uvAtAD0Z+D57kn5L8C2wB0DPeyQYURuBAYQQOFEbgQGEEDhTW5NDFCy85rOnpnS0eemy0OnRy+sB4fV7H6fDNcfrcrt94uNN13MGBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcKanKraCieVjtcppeOm4tcXd3CgMAIHCiNwoDACBwojcKAwAgcKI3CgsM6B256w/Y7tl1sOAtCfk7mDb5a0p9UQAP3rFLjtVZJukPRk2zkA+tT1Dv6opPsl/XCiC2xvsj1je2b20LFexgFYnJGB275R0pdJ3l7ouiRbkkwlmVpx7kRvAwGcui538Ksk3WT7U0kvSLrG9rNNVwHoxcjAkzyYZFWSNZJukfR6ktuaLwOwaPwcHCjspH4fPMmbkt5ssgRA77iDA4UROFAYgQOFEThQGIEDhY3VqaqtTqfkpFK01OLra28OdbqOOzhQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UNhYnarK6aftTpYdN3wtdMMdHCiMwIHCCBwojMCBwggcKIzAgcI6BW77bNtbbX9ge4/tK1oPA7B4XX8O/pikV5P8xfZpkiYbbgLQk5GB2/6tpKsl/VWSkhyRdKTtLAB96PIUfa2kWUnP2H7H9pO2z2y8C0APugS+TNJlkp5Icqmk7yQ98NOLbG+yPWN7ZvbQsZ5nAjgVXQLfL2l/ku3zf96queB/JMmWJFNJplacO9HnRgCnaGTgSb6QtM/2RfMf2iBpd9NVAHrR9VX0uyQ9N/8K+seS7mg3CUBfOgWeZKekqcZbAPSMd7IBhRE4UBiBA4UROFAYgQOFEThQ2FidqsqJou1OEx23z+247e3b+o2HO13HHRwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwsbq0MVWWh1kOE7G7XPQ4tDFcfoc7M2hTtdxBwcKI3CgMAIHCiNwoDACBwojcKAwAgcK6xS47Xtt77L9vu3nbZ/eehiAxRsZuO2Vku6WNJXkYkkTkm5pPQzA4nV9ir5M0hm2l0malHSg3SQAfRkZeJLPJT0s6TNJByV9neS1n15ne5PtGdszs4eO9b8UwEnr8hT9HEk3S1or6XxJZ9q+7afXJdmSZCrJ1IpzJ/pfCuCkdXmKfq2kT5LMJvle0jZJV7adBaAPXQL/TNLltidtW9IGSXvazgLQhy7fg2+XtFXSDknvzf87WxrvAtCDTr8PnuQhSQ813gKgZ7yTDSiMwIHCCBwojMCBwggcKKzJqap7350cqxMqW2hx6qc0Xid/SuP1eWi1tYX1Gw93uo47OFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQmJP0/6D2rKR/d7j0PElf9T6gnXHaO05bpfHauxS2/j7JilEXNQm8K9szSaYGG3CSxmnvOG2VxmvvOG3lKTpQGIEDhQ0d+JaB//sna5z2jtNWabz2js3WQb8HB9DW0HdwAA0NFrjt62x/aPsj2w8MtWMU26ttv2F7t+1dtjcPvakL2xO237H98tBbFmL7bNtbbX9ge4/tK4betBDb985/Hbxv+3nbpw+9aSGDBG57QtLjkq6XtE7SrbbXDbGlg6OS7kuyTtLlkv62hLceb7OkPUOP6OAxSa8m+YOkP2oJb7a9UtLdkqaSXCxpQtItw65a2FB38PWSPkrycZIjkl6QdPNAWxaU5GCSHfN//63mvgBXDrtqYbZXSbpB0pNDb1mI7d9KulrSU5KU5EiS/w67aqRlks6wvUzSpKQDA+9Z0FCBr5S077g/79cSj0aSbK+RdKmk7cMuGelRSfdL+mHoISOslTQr6Zn5byeetH3m0KNOJMnnkh6W9Jmkg5K+TvLasKsWxotsHdk+S9KLku5J8s3Qe07E9o2Svkzy9tBbOlgm6TJJTyS5VNJ3kpby6zHnaO6Z5lpJ50s60/Ztw65a2FCBfy5p9XF/XjX/sSXJ9nLNxf1ckm1D7xnhKkk32f5Uc9/6XGP72WEnndB+SfuT/P8Z0VbNBb9UXSvpkySzSb6XtE3SlQNvWtBQgb8l6QLba22fprkXKl4aaMuCbFtz3yPuSfLI0HtGSfJgklVJ1mju8/p6kiV5l0nyhaR9ti+a/9AGSbsHnDTKZ5Iutz05/3WxQUv4RUFp7inSLy7JUdt3SprW3CuRTyfZNcSWDq6SdLuk92zvnP/Y35O8MuCmSu6S9Nz8/+g/lnTHwHtOKMl221sl7dDcT1fe0RJ/VxvvZAMK40U2oDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwr7H1tiS3kfJfL7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "size = (10,10)\n",
    "J = np.prod(size) \n",
    "hid_layer = 10\n",
    "layers = [(2,hid_layer),(hid_layer,2)]\n",
    "params = gen_params(1,2*hid_layer+hid_layer*2)\n",
    "grid = init_grid(size=size)\n",
    "grid_ = grid.clone()\n",
    "grid__ = grid.clone()\n",
    "plt.imshow(grid)\n",
    "print(grid.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'size' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-ebfeadc4a688>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.0001\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnum_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m3\u001b[0m \u001b[0;31m#A\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m#B\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mreplay_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m \u001b[0;31m#C\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mreplay\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeque\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreplay_size\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#D\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'size' is not defined"
     ]
    }
   ],
   "source": [
    "epochs = 75\n",
    "lr = 0.0001\n",
    "num_iter = 3 #A \n",
    "losses = [ [] for i in range(size[0])] #B\n",
    "replay_size = 50 #C\n",
    "replay = deque(maxlen=replay_size) #D\n",
    "batch_size = 10 #E\n",
    "gamma = 0.9 #F\n",
    "losses = [[] for i in range(J)]\n",
    "\n",
    "for i in range(epochs): \n",
    "    act_means = torch.zeros((J,2)) #G\n",
    "    q_next = torch.zeros(J) #H\n",
    "    for m in range(num_iter): #I\n",
    "        for j in range(J): #J\n",
    "            action_mean = mean_action(grid_,j).detach()\n",
    "            act_means[j] = action_mean.clone()\n",
    "            qvals = qfunc(action_mean.detach(),params[0],layers=layers)\n",
    "            action = softmax_policy(qvals.detach(),temp=0.5)\n",
    "            grid__[get_coords(grid_,j)] = action\n",
    "            q_next[j] = torch.max(qvals).detach()\n",
    "        grid_.data = grid__.data\n",
    "    grid.data = grid_.data\n",
    "    actions = torch.stack([get_substate(a.item()) for a in grid.flatten()])\n",
    "    rewards = torch.stack([get_reward_2d(actions[j],act_means[j]) for j in range(J)])\n",
    "    exp = (actions,rewards,act_means,q_next) #K\n",
    "    replay.append(exp)\n",
    "    shuffle(replay)\n",
    "    if len(replay) > batch_size: #L\n",
    "        ids = np.random.randint(low=0,high=len(replay),size=batch_size) #M\n",
    "        exps = [replay[idx] for idx in ids]\n",
    "        for j in range(J):\n",
    "            jacts = torch.stack([ex[0][j] for ex in exps]).detach()\n",
    "            jrewards = torch.stack([ex[1][j] for ex in exps]).detach()\n",
    "            jmeans = torch.stack([ex[2][j] for ex in exps]).detach()\n",
    "            vs = torch.stack([ex[3][j] for ex in exps]).detach()\n",
    "            qvals = torch.stack([ qfunc(jmeans[h].detach(),params[0],layers=layers) \\\n",
    "                                 for h in range(batch_size)])\n",
    "            target = qvals.clone().detach()\n",
    "            target[:,torch.argmax(jacts,dim=1)] = jrewards + gamma * vs\n",
    "            loss = torch.sum(torch.pow(qvals - target.detach(),2))\n",
    "            losses[j].append(loss.item())\n",
    "            loss.backward()\n",
    "            with torch.no_grad():\n",
    "                params[0] = params[0] - lr * params[0].grad\n",
    "            params[0].requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x12591eac8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAJCCAYAAADdrPONAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsvXl0JHd57/39dVdX7y21ulsajaRZpbFnPIs3bAM2BC8sttlMEggkueG+gSy8Jtu9SXwuCcm9F8IlBAjknPtewg1JCMtJWAJ4w9gQ8IIN47Fn83hGM54ZrSOpJfW+VVXX+0f1r7sl9VJbd1dJv885HEBqtUo93VVPfZ/v832ILMtgMBgMBoPBYLTG0esDYDAYDAaDwbADrGhiMBgMBoPBUAErmhgMBoPBYDBUwIomBoPBYDAYDBWwoonBYDAYDAZDBaxoYjAYDAaDwVABK5oYDAaDwWAwVMCKJgaDwWAwGAwVsKKJwWAwGAwGQwVcJ540Go3Ku3bt6sRTMxgMBoPBYJjK888/H5dlOdbucR0pmnbt2oWjR4924qkZDAaDwWAwTIUQclnN41h7jsFgMBgMBkMFrGhiMBgMBoPBUAErmhgMBoPBYDBUwIomBoPBYDAYDBWwoonBYDAYDAZDBaxoYjAYDAaDwVABK5oYDAaDwWAwVMCKJgaDwWAwGAwVsKKJwTAJUSrj9Fyy14fBYDAYjA7BiiYGwyQee2kB937+KVxezvb6UBgMBoPRAVjRxGCYxEKqAFkGXr6S7vWhMBgMBqMDsKKJwTCJdEEEAJxfzPT4SBgMBoPRCVjRxGCYRCovAAAuLLGiicFgMDYjrGhiWAZZlvGp75/FMxfivT4UXVCl6cIS8zQxGAzGZoQVTQzL8A9PX8Lf/eg8vvPCXK8PRRfpYkVpWsxAluUeHw2DwWAwzIYVTQxLcHImiU88cgZArfiwG6m8ojRliiIW08UeHw2DwWAwzIYVTYyekymKuP9rxxANuDExGKgWH3YjXRDgcSkfqQvMDM4wkaIooShKvT4MBmPLw4omRs/5838/hamVHP72PddhNOxFqmBPpSldEHF4pB8AcJ6ZwRkm8rv/cgy///UXe30YDMaWhxVNjJ7yzedn8K0XZvHhOyZw0+4BhLyu6hSa3UgVBOwdDCDg5pjSxDANWZZx9PIqnj4fZ145BqPHsKKJ0TNeWcrgz75zCjfvHsD9t08AAIIeDqmCPdtzqYKIkJfD3pifTdAxTCOeKSGZF5AqiLi0nOv14TAYWxpVRRMh5PcIIacIIacJIb/f6YOyO9MrOXZH2IaiKOH+r70AnnPgs++5Fk4HAQCEPIrSZLfXryBIKIllhDwu7B0MsIDLBiymChCkcq8Pw3bUv5eOTyd6eCQMBqNt0UQIOQjgAwBuAnAEwL2EkPFOH5hdmV7J4XV//SM8fX6514diaT7xyMs4PZfCX//iEQz3eatfD3ldEMsy8oK9TK80oynk4bA3FsCVVAGZoj0Vs06QL0m4/W9+jK/9bKrXh2I7zi8qa3kcBDg+w4omBqOXqFGa9gN4TpblnCzLIoAfA7ivs4dlX5YyRcgycIktbW3KE2cW8KWnL+E3XrMLdx0YWvO9kMcFoFaE2IV0xbwe9LiwNxYAoLQfGQpTKzlkiiJeYW1LzZxfzCDg5nDdjjBOzCR7fTgMxpZGTdF0CsBthJAIIcQH4G4AY509LPtSKCkKSTzDcnoacSVZwH/5t+M4MBzCA3dfveH7IS8HAB0xgxcECQ986wRmVs33hVAfVsjLYXzQD4CtU6nncuUmgn0utDO5mMHewQCOjPbj1GyStTgZjB7StmiSZfkMgP8F4DEAjwJ4EcCG3gkh5IOEkKOEkKNLS0umH6hdoG2l5Uypx0diTT7y76dQFMv4/Huvg5tzbvg+VZo6ETvw1GQcX/vZNJ7pQOu0XmnaGfGDcxDma6pjakUpVJdY6Kdmzi9mMDEYwJGxPhTFMs4tpHt9SAzGlkWVEVyW5f8ry/INsiy/DsAqgHMNHvMFWZZvlGX5xlgsZvZx2oaCoNwFsjvqxrw0l8SbD26rtrDWE/JWiqYOBFw+dV7ZaZctmf/ctJ0Y9HBwOR3YEfHhwiJrRVEuV6a+ltjnQhPJnIDFdBHjFaUJAGvRMRg9RO303GDlv3dA8TN9tZMHZWeY0tSadFGsqkmNCHoq7bkOKE1PTioKaK5kvsmcthPp37Y3FmDtuTouM6VJF+eXFFVpYjCAnREf+rwuNkHHYPQQTuXjvkkIiQAQAHxIlmX2qW1CoVI0xbPs4rAeWZaRLYrVwqgR1facyZ6m+WS+mp2U7cBUW73SBADjgwH8x9lFiFIZnJPFoU1VPE3pgoiCIMHj2tiaZWyEtngnBoMghODwaB+OM6WJwegZattzt8myfECW5SOyLD/R6YOyM9Wiid1RbyAvSCjLQMDdvGiqKU3mFjZPTsar/7sjSlNBACGAn1eOf28sAEGSq16erYwolTGzmkc04AbA1CYtTC5k4OYcGAkrsRxHRvtxbiGNfAfewwwGoz3sFthkaNGUKogoiWzKpZ5MpRDytyiaPC4n3JzD9Pbck5NxxIJuDPd5OpKflC6ICLo5OCohnXtjdIKO+ZrmkwWIZRk37FQ8OczXpJ7zSxnsjQWq4a+HR/sglWWcnmNqE4PRC1jRZDL1oYzLrEW3hnRxbQurGcr+OfMKm3JZxtPn47htPAq/m0OuA0bwVEFAsM6rtXdQMbqzCbra5NyNOwcAMKVJC5MLGYwP1oYmrh1TCk/WomMwegMrmkyGTs8BzAy+HuolatWeA5RUbTOVppfmU1jJlnDrRBR+3olssRNG8LVerZDHhcGgm5nBUZucu2FXGAArmtSSK4mYTeQxUVc0DYY82BbyaDaDvzC1ioVUwexD7Ar5koTvn75iu9VKjM0JK5pMpl5pYm2ItdD2XLuiKVjZP2cWNGrg1vEofHxnlKZ0QajGJVDYBJ3C5ZUseKcD12wPgRBWNKmFRlbUK00AcGSsDyc0rFPJFEX8yt8/i49+57Spx9ctHj09j9/68vN4+OSVXh8Kg8GKJrMplCRU7AdMaVoHbc+18jQBlfaciUbwpybjuGooiMGQB353Z5SmdEFEaF3bcbyyuHer3yFPLecwOuCFm3NiwMezDDOVTFZ2zk0MrS2aDo/249JyDomcuvPL4y8toCCU8aOzi7bch0jPo599/Byk8tb+LDF6DyuaTKYgStgW8gAAltnFYQ2ZgkpPk4dD2iSlqSBI+NmlFdw6EQWAjilNqYKwIX9qb8yPdEHc8orj5eUcdg74AACxoJspTSo5v5gB5yDYGfGv+brWkMsHT8yB5xwoimU8cWbB9OPsNKuV4nByMYMHT8z1+GgYWx1WNJlMviRhIMDD43KwO+p10CTutp4mr8s0T9PPL62gJJarRZPf7US2A+Pa6cLG/ClqBt/KyeCyrMQu0At/LOje8kWkWiYXM9gV9cO1Lufr0GgfAKhq0SXzAn58bgnvu3kHhkJuPHRiviPH2kkSOQFhnwtXbwvis49PQmS79xg9hBVNJlMQyvC6nIj43aw9tw4aABloqzSZ1557cjIO3unAzbuVyS0/z5kebinLMtLrpueAmhfl/Bb2Na1kS8gUReyoKE3RAFOa1HKhsnNuPX1eF/ZE/Xhxur3S9NjpKxAkGW87sh13HxrGf5xbqu5JtAuJnICwn8cf3LUPF+NZfPuF2V4fEsMkZFlGMmev9yMrmkwmX0k7jrI76g1kiiJcTtJwUW89IS+HkliuZl4Z4cnJOK7f2Q9fJXTS5+aQK0kom+iNyJaU0M6Qd20xuC3kgY934sIWjh2g61N2Rta257a6z6sdRVHCpeXsBhM45fCoOjP4gyfmMRr24tqxftx7eBglsYwnziyafbgdJZEvod/rwhsPDOHgSAif++EkBKY2bQr+/slXcOsnf4iiaJ+wVlY0mUxBkOB1ORH180xpWkemILZtzQF1q1QM3hEvpYs4M5/CbRO1BdJ+XinY8iYUZBR6575eaSKEbPkJuqnldUVTwI2iWK4OBTAacymeQ1neODlHOTLWj8V0EVeSzWMEVrMlPH0+jnsOD4MQguvGwhju8+BBm7XolPYcD0II/vCufZheyeMbz8+o/vmCIOHUbJKlqFsMWZbx1eemkC6IiNvoWsmKJpOhe7WiATcLt1xHtii2bc0BdatUDAZcPnOhFjVA8VWKtqyJZnB6nI0M7uODga2tNC3nQAgwGq4pTQCLHWhHdXJuMNjw+4crZvAXW+Q1PXr6CsSyjLce3g4AcDgI7j40jJ+cW9J0QyLLMh46MY+VbG8ubImcgD6fckPyhqsGce1YPz7/xKQqdSJbFPHev38W937+KRz6i+/jrZ9/Ch/9zil858VZTC3nmOLZQ56/vIpLlZsqOw1NsaLJZKqepoCiNJnZBrI76aKIgNvV9nE078io0vTkZBx9XhcOjvRVv0aVppyJsQNUaVo/PQcoE3RzyUJHlgTbgcsrWWwLeaoLelnRpI7zixkQAuyJ+Rt+/5rtIXAO0rJF9+CJOeyO+nHN9lD1a3cfGkZJ0jZF98yFZXzoq8fwl9/rTc5TIldC2McDUNTbP3rjPswlC/jXn0+3/LmCIOED/3wUx2eSeOAtV+O3Xr8HATeHf3t+Br/39Rfxur/+EV71scfxgX8+iu+8aJ5PKlMUcfTSCv7pmUv4428cx1s//xT+/ievmPb8m4VvHquphXbqyrS/7WdoQvE0ORAJuCGWZaQKAvorH/itjtKea7/dvtqeMxA7IMsynpqM47XjkereLgBVb5OZSlO6RZTC3pjSXrkYz64p3rYKU8u5qgkcYEWTWiYXM9gx4KsWm+vxuJy4aluwaezAUrqIn15YxofeMA5Cau//68b6sb3Pg4dOzOOd1422PQ5ZlvGZH5wDAHz3+Bx+9xfGcdW2xupXJyiJZWRLEvrrgmNvHY/ipl0D+Lsfnccv3TjW8DUSpDL+368ewzMXlvHpXz6C+66v/a2iVMbZhTSOTSXwwuVVPHk+jmcvLOPt147oOsZL8SwePjWP03MpvDSXwqXlLKiANeDnIUhlPHhyHh943R5dz78ZKQgSHjw+j1ftCuPnl1ZtNWnOlCaTKQgSPLwT0YBSKNnpzdBpMkV1nqa+iqHayATdhaUMrqQKuHU8tubr/krRljPR35Bq4mkC6ibotmiL7vJKrupnAhRPE8CKpnacX8hgPNbYz0Q5PNqP4zOJhmr2I6fmUZaBeyutOUqtRRdHUsVNyVPn4zh6eRV/dNc+BHgOn/7BWW1/iEESeUWB6PfVPluEEPzBXfuwkCriK89NbfgZqSzjD//1OB4/s4j/8Y6DawomAOCcDlyzvQ+/dstOfPrd1+I3XrML6aKoe/Dkz75zCp989CyOTyewbyiAP7hzH7746zfipw/cjuc/cifuOTSM2dWcruferDz20gLSRRG/9bq9AIDlHrV+9cCKJhMpl2UUxTI8nOJpAmArg1unUTxNKtpzlccYGY1+clLxM902EV3zdZpGbmYyMi3u1k/PAcCOiA9OB+mKGXwxXcDFuHUyoXIlEUvp4ppwxj6vCy4nYTcTLRClMi7Gsxgfal00XTvWh3RBxKXljf/mDx6fx8RgoKEqdM9hpUX3+EutW3SyLOPTPziH7X0efPD1e/Cbt+3B908vaFrhYhQ6jr5erX/13gheszeC//0f59eE1cqyjP/27ZP43vE5PPCWq/Frt+xs+zuM3uBeSRbwpmuG8NSf3I7/82s34sN3TODOA0MY7vOCEILRsBfxTIkZ0ev41rEZbO/z4ParB+HjnYjb6CaKFU0mUqgYE718rWiyU6+206RVKk1VT5MBI/iTk3HsivgwVtcaApScJqB7niY358SOAV9XiqY///fT+O0vP9/x36OWqUrcQH17zuEgLKupDVMrOZSksiqlCdiYDH4lWcDPL69sUJko1471Y6Tfi4dOtp6i+/G5JbwwlcCHbh+Hm3PiP9+6C2GfC5967JyGv8YYq9WiaeNn64/euA/xTAlf/ullAErB9D8ePIOv/3wa998+jt96/V5Vv8PoDe5ytlR9jkbQIYjZhD3VpkvxrCpVUi2LqQJ+cm4J77x+BA4HUfy/NlKamKfJRAqCkh3i4RyIsPbcBtR6mtycA7zTodsIXhLLePaVZdx3/UaPgq9iBDd7eo53OuDmGt+D7I0FupIK/uJ0wtS/yyiX18UNUFgqeGtoK3diqLV3aGIwAI/LgRenE3jHdbX3+kMn5yHLwL1Hhhv+HCEE9xwexpeevohk3WRaPbIs4zOPT2Kk34tfumEMgNJ+/u3X78VfPfIyfnZxBTdVAmM7Cd2v1+/d6Au9YecAXr8vhv/vxxfwvlt24gs/voB/ePoi3v/aXfjDu/ap/h2R6g2u9vekKJWxmmtdNI0NeAEA06t5jDeZhrQqp2aTuPfzTwFQisvxQT/2xgIYHwxU/3u4z7PGN9eOf39xFmUZ1bZpxO+21XWSKU0mQrN/vLwTYR8PB7HXKGUnEaUy8oKkanqOEIKgh9NtBH9hahW5krTBzwTU2nM5E9tzSho41/TEsXfQj4vxbEfXPyyli7iSKiBdEC0TFFfNaBpYOwG2FZWmfz06jb/4rrrps8lK0dQso4nCOR04NLIx5PLBE3PYPxyqDiE04u5DwxAkGY+9dKXh9390dhHHpxO4//Zx8HU3A7/+6l2IBd341PfPdmVcP9FCaQKAP7hrH1ZzAt73xefwuR+ex7tvHMOf33tA00XcSHtuJVuCLNeeoxFUaZpZzWt+/l5DW7//z627cfvVMZTEMr53fA5/+b2X8Ov/8DO85hM/xE0ffwKTC2lVzyfLMr75/Cyu29FffX9GA/bKNGRFk4lQI6HH5YTTQTDg57FkozdDJ6H73tTkNAF0/5y+wuap83E4iOJ7WE9NaTKzPbdx71w9e2MBlKRyR0+ap+dqLZpe5ems5/JKFn1e1wYlI7bFiqZ4poj//r2X8I/PXMLZK+0vLhcWMxju86hqZR8e7cfpuVQ1IXtmNYcXphJ4axOViXJktA8j/V483KBFp0zMTWJswIt33bDWRO3lnbj/9nH87NJK1TfYSRoZweu5dqwfd+4fxPHpBO49PIyP33dIU8EEGGvPUcW0ldIUC7jBOx2YWbFfe44WM7/zC3vxyV88gm/97mtx/KNvxM//2534+gdvwf98x0GIUhn/9RsnIKmI1zk9l8LZhTTeVWfOZ0rTFoYa/egIrLJ/zj5vhk5CjddBFRcCAAgZUJqenIzjyFg/+ryNPEYOOB1kjXnUKKmCUPVhNYIqBp30NZ2eS1X/dzxtkaJpObehNQco7bnlbEnVSXYz8PknJpEXJPBOB7763OW2j59czLRVmShHxvpRFMvVYowu5L33UGM/E4UQgnsPD+PJyfiG3V+Pn1nEydkk7n/DxIZlwQDw7leNYaTfi0891nm1KZETwDlIywLyv7/9ID5yz3585t3XrokXUYvH5UTAzem6cNNCKxpsXjQ5HAQjYa8tlaZ4pggHQTUnC1DeO7GgG7fsieBXb9mJv3jbNXhxOoEvPX2x7fN94/kZ8JyjGrgKANEgj5WsfTINWdFkIrQtQoumaJC3VQXdSTIV1civtmjyunR5mpI5ASdmErhtPNrw+4QQ+HgnsqYawdsoTdHOxw6crDMDxy2SRD+1sjajiRILuiGVZazmrFHcdZKL8Sy+8twU3vOqMdx9aBu+dWy2ZdBpuSzjvJaiaVTJ/qJm8AdPzOPIaB92NChW13PP4WGIZRnfr2vRybKMzz5+DjsjPryzgScQUIYbfu+OCZyYSeKxNhN4RlnNCej3uVqqR9v7vfjN2/Y0LPDUEtHZIlpWoTQBwGjYixkbxg7EMyUM+PmWxejbjmzH7VcP4lOPncXlBpOclJJYxnePz+Gu/UNr1OeIv5ZpaAdY0WQi+ZIikXvrlSaLtEp6TaaofCBUt+c8Ll1K009fiaMsA7dObPQzUQJuztSE7lReQLCFV6vP50I04O6o0nRqLolDlfBMK/gDRKmM2dV8U6UJ2BpZTX/9/ZfBcw783p0TeN8tO5Euivje8bmmj59L5pEXpKbrU9azY8CHfp8Lx6cTuBTP4uRssunU3HoOjfRhbMBbVacAJT/n9FwK99/eWGWi3Hf9CPZE/fj0Y+c6qhgm86WGirHZRAP6WkTxatHUOsB4NOyzrdIU8bcuCAkh+Ng7D8LlcOBPv3myqfr4H2cXsZIt4V03rC3Ga0NTvT9vqYEVTSZCPU20aIoG3Ja4gFmBTEXZUePTAJTMo7QOT9NPJuPw805ct6O/6WN8vNPUcMt0QWyY0VTP3pgfF5Y6M0GXyJUws5rH6/cphaIVWsJziQLEsrzBBA5snaLphalVPHzyCj74uj0YDHpw484w9g0FGgYyUtSawCmEkGrI5YMnlGLsnsOt/Uz1P3vPoe14+nwcq5X2yGd+cA67o36849rWhRfndOD379qHswvp6u/tBHRZb6eJ6FywHs+UwHOOtue10bAXy9mSqbaAbrCcKSIabP/6D/d58cDd+/HTV5bx9Sbrbb55bAbRAI/XrbuhrXnK7HE+UFU0EUL+gBBymhByihDyNUKIp9MHZkfyVSO48rJGAjwyBpJmNxOZFqtGGhH06GvPPTUZxy17Ii3vkv1uzuQ1KkLDNPB6xgcDOL+Y6YgHhPqZbto9AI/LYQl18/KKUiA2ahNthVRwWZbxVw+/jGjAjQ/cpqzPIITgV2/ZiZOzyaYBkXS584TKoglQWnSTixl869gsbtgZxvZ+r+qfvbfSonvspSv4/ukrePlKGh++YxycilbXvYeGcfW2ID77+GTHJkNpe67TRIM6laZ0EbGAu635fDSs/JvM2kxtimdKbZUmyq/cNIZX74ng4w+dwXxy7d+5mi3hhy8v4u3Xjmx4b9kt07DtJ4MQMgLgwwBulGX5IAAngPd0+sDsSF5Y52liWU1VaHtOtafJw6EglDWNz69kS5hayeHmPa3zY3y807RwS1FSdmO1Kwb3xgJI5oWOFDSnZhU/y8GRPstMojTLaAJqSpMVjrNTPH5mET+7tILfv3NizXv+HdeNwOty4ivPNlabJhcyiAZ4hP3q1ZXDo/2QyjJeiWdxr0qViXLN9hB2DPjwvePz+Ozjk9gT8+NtR9TtYHM4CP7ojVfhYjy7ZvmqmSRzpa7s7oz6eazktA8nxLOltq05oBY7MG0zX9NyptjWr0UhhOAT7zoEoVzGR759as0N4nePz0GQ5DVTcxTanlu2iBezHWrbcxwALyGEA+AD0Dk91sYUNxRNbJUKhbba1Lfn6CoV9YoQVS7a3Wn7efOUJjoV2CgNvJ69dIKuA2bwU3MpjPR7MeDnLZN5MrWSA885MBTcKEr73Rx8vHPTKk2iVMYnHjmDPTE/3v2qsTXfC3lcePu12/Gd47MNU5bPL2Va5is1gprBCQHuOaStaKJBl0+dj+PsQhq/d8eEpgm0O/cP4shYPz73xPmO5IOt5oQ1y3o7RTTohixD83BCPK2uqBirKE128jXlSxKyJala1KhhZ8SP//LGq/DEy4v4bp1375vHZrB/OIQD20Mbfibs40GIfa6TbYsmWZZnAXwKwBSAeQBJWZYf6/SB2ZH6cEvAWNLsZiOr1dPkoatU1Lfo4ionWXxuzjRPU1pl27G6uLcDZvDTs0kcHFFORpGA2xJ3bJeXs9gx4IOjyQV4M6eC/9vzM7iwlMWfvPnqhm3i9928EwWhjG+vU2dkWcbkQhoTbXbOrWcw5MFo2ItbdkcwGNLunKCF1vhgQLWJnEIIwR/dtQ+ziTwePdU4KFMvBUFCXpC60p6jLSit6mc8U1RVVEQDbvCcw1ZFE30tYiqVJsr7X7sb14714y++exrxTBGTC2mcmEniXU2mMZ0OggGffSbN1bTnwgDeDmA3gO0A/ISQX23wuA8SQo4SQo4uLS2Zf6Q2oH6NClBrz1nhzr/XZIoCfLxT9V0sNVZrCbhUO8ni552mTc9RtaBVThMADIc88Lqcpq9TSRcEvBLP4uB2RW3Qa2g1m8vLOexsEDdA2ayp4LmSiE//4Bxu3BnGGw8MNXzModE+HBntw1eem1rTwljKFJEqiG13zjXiH37jVfibXz6i65iv2R7Cb966Gx97x0FdOUevHY/C63LihSlzF/nSG6autOd0nKvLZbnt3jmKw0FsFztAz6dalCZAKYI++YuHkSmK+IvvnsY3j83C6SB4+7XN277K0JQ9zgdq2nN3Argoy/KSLMsCgG8BeM36B8my/AVZlm+UZfnGWKz5uPdmJi9IcDlJ1ehG71426x21FjJFUbWfCdCnNNGLcLuTmN/EyAG1SpPDQbAn5jc9duClign8YCVuIFKZ2DTbcF4Sy7j380/ik4++3PaxsixjaiW3YVlyPZs1FfyLT17EUrqIB+6+uqU5+H0378TkYgY/v7Ra/dr5BXU75xqxbyioyQBeDyEEH7n3AG7eszFBXw1OB8GB7aGqt84sWi3rNZuIjgmuRF6AVJZVe37sFjtAC0i1f189+4aCuP/2CTx4Yh7/9Mwl/MK+WNXL2Ai9OVm9QE3RNAXgFkKIjyhngTsAnOnsYdmTgiDBw9UW0np5J/y80zZvhk6SLoiq08ABfZ6meKYEl5O0zXXx807kBMmUBFo64dfO0wTUJujM5FSlaLqm0p6LBniUpDLSJuZQAcCXn72MU7MpfPVnUyiJrSel4pkSciWpoQmcshnbc/FMEf/nxxfw5mu24YadrYcR7j0yjKCHw788W0sI1xo3YCUOjfThpfmUqZlNdFlvNyIHYjr8p9VgyxbFQD2jYS+mbbRKhbb5tSpNlN/5hb24elsQeUHasI5nPYqtwB7XSTWepucAfAPAMQAnKz/zhQ4fly0pCBI8vHPN1/SOsm42skVRdbAlUFNutMQO0CC2duO/PjcHWQYKJhhXaVGnqmiKBTCbyOONn/kxPvDPR/Hxh8/gq89N4ZkLccwl8rqKuNOzSQwG3RisGK6rQXEmqjir2RL+9vFz2BbyIJET8KOziy0fP1WJG2hXNCVygmWWC5vB556YREEs44/ffFXbx/p4Du+6fhSPnJrP2/nHAAAgAElEQVSvnh/OL2YQ9HAYVHkRthLXbA8hV5JwMW7eTQFVmroRbhnycnA5iaZz9ZJKOwBlNOzFak6oDo9YnbgBpQkAXE4HPvcr1+F9N+/AHfsHWz424udNPWd1ElVXMVmWPwrgox0+FttTEMrVjCZKxM9bwpjbazJFUbUJHNDXnlMbxOanS3uLEny8+mNqRLpS1KnJn3r3q8aQKYp4JZ7FpXgWPz63tEa18bgcuGl3BP/4G69qaqBez6m5ZLU1B9RawsvZEvaY1CX/2ycmkSmK+PoHX41f/4fn8O1js3jTNduaPp7GDexoEGxJoVL9cqaku61kJV5ZyuCrz03hvTftwB6VnqT33bwD//jMJXzj+Rn89uv3YnIxjYnBgOaFs1bgUGWC7+RsEuMq08zbkWyzrNdMCCGad4VqLSpo7MDsah5XbVP/Gv3Wl4/insPb8bYj2kz6Rolnigi4ueo0uB72DQXxsXceavu4aIBHupJpaOT3dQNjVwzGGvIlqZoGTokG3NWLyFYmXRBbelzWQ03j2pQmdaZMWigp6bzG7upT+UqUgoqiaTDkwQN376/+f6ksYz6Zx+XlHC7Gs3ju4gq+d3wOx6ZWceOu1u0dQHm/nV/M4M0Ha2Pm1cwTk9TN84tpfPnZy/iVm3bgwPYQ3npkO77y7BSSOWHN/qh6Li/nQAgwNtC8GKoPuNwMRdO/Hp0BIcCH75hQ/TMTQ0HctHsAX31uCh+8bQ/OL2Zx+9X29IOOxwLwuBw4OZPCO68z5zkTFaWpG+05QPnsaGnPxVV6KCm12IGc6qIpnini+6cXMODne1A0qcugMgP6Gq5krX8TxdaomEhB3Fg0WWUEvNdkito8TYQQhDxctShRQ1xlEJvfXVOajJIuKFOBepaFOh0Eo2EfXjsexa/eshMff+dB8E4HHj6pbnT7zJUUyjJwsC77xOxssI89dAY+lxN/eNc+AMB9142iJJXx4MnmUW1TKzkMhzxwc83vGDfbKhX63mtldm3E+27egamVHL53Yg7xTFH1zjmrwTkd2D9srhl8NSfA5VQWbHcDrRNcy9kinA6iOkeKKk1azOBn5hXP4mq2+8tslzPFqkG+09Tieazva2JFk4nkSxLc64qmWIDHSlZ70uxmQ6unCVDM4GqVJlmWsaxLaTJGuiCqXg3TjqDHhdfti+KRU/Oq/E31SeCUAb95MRc/ObeEH51dwv13jFdPagdHQhgfDODbx2ab/tzl5WzD9Sn1bLZU8GRe0OW9efPBbYj4efyvR5SpRDuawCmHRvpwei5pyoAFoLTn+n1819qV2pWmEiJ+XnUrPRrg4eYcmmIHqkWTxtBNM1BuBLqn8tHfaXVY0WQiBaGx0lTWkTS7mZBlWbOnCVB8TWo9Tam8iJJUVvUhp0qTGYbMlIq9c1q4+9Aw5pMFvDDdPvPm1GwSA34ew321QEOX04F+n8uwuilKZfzPh17CzogP/+k1u6pfJ4TgndeN4OjlVUw1aTtPreQaLuqth54kN4vSlMwLbbO6GuHmnPilG8cwlywAsHfRdHCkD9mShIvL5mSRrWa7kwZOiQWUoR21cR1xjUoMIaQyQadFaUoDqLUqu8lyptQ1pSmqM1y0F7CiyUQaGsFZwCWKYhmCJGvKaQKUiRa1kQO1SRY17TmqNJkzPRcySWkCgDsPDMHlJHjk5Hzbx56aTeGa7aENd+JmBFx+/efTOLeQwQNvuXpDm+0d1ykhdd9+YaPalC2KiGdKbZUmN+dEv8+1aWIHUnn9F/j33rQDAOB1OTFicT9HKw5VFE+zWnSJfKkrJnBKJMCjKJZV30zpUWJGwz7MJLQrTYl8d68folTGSk6dcm8GdIDHDrEDrGgykXwDpSmqIzRts0FPQlrbWEG3+vac2hUqgLJ7DoApAZdpk5WmkMeF2yZieOTUlZZ3vEVRwrmF9JrWHCUSMBZzkSoI+PQPzuHm3QMNp+RG+r24Zc8Avv3CzIZjnFppvqh3PZspFVxvew4AdkR8uOvAEI6M9alu9ViRicEA3JwDJ2dMKppyQlfSwClRjb6aeKakecXI2IBXtaepKCqDHoQo/i6zA2tbofw+9XEKRvHxHLwupy1SwVnRZCKNxiWjNurVdgpanGhuz3nVG8GrRZOKyAFqLDVDaUoVRF1tmVbcfWgYs4k8XmzRojt3JQOxLFfv7uuJBnhDd2x/98PzWM2V8Gf3HmjqJ7nvulFcWs5taCPSSdF27Tlgc6WCGymaAODv3nsd/vH9N5l4RN2HmsFPmqU0dWlZL0VLKrgsy4rSpNH4Pxr2IZETqlElrTi/qHzGDwyHUBLL1d2m3UDLTahZaPWU9QpWNJlIvmHRZJ+pgE5BW2y6PE1qlSYN47+0PZc1xQgumGYEp9y1v9Kia7EA9dRcxQS+vYHSpDFvpp7Ly1l86emL+MXrRxuqWJS3HNoGN+fYYAinwZbt2nPA5kkFF6QyciXJUNHk5pyWz6dRw8GREE7PpUwxg3e7PVe7wW1/rs4URRRFdR7KekYrsQOzifZqE/UzvWavst6mm74mer2K+Lun9BlVyLsFK5pMpCiUN5z4Qh4XOIe2pNnNRka30uRCriRBkFqv7QCUXriDqMt0cXMOOAiQMyFyIJU3b3qO0udz4bXjUTx0Yr6pJH9qNomgh2uYhRQJ8FjNCRBVvG7r+auHX4bL6cB/fVPrVOugx4W7Dgzheyfm1gR0Xl7Ood/nUlVAxIKbQ2miS5ub5VZtJQ6N9CFTFHHJoBm8IEgoCOWetOfUnKvj1aJCu9IEADMqzOBn5lPwuBy4bkcYQHeHiWrKffeUpphN9s+xoskkpLKMklTe4GlyOAgGLLJ5vldkCuoDIOuhBms1ZvB4pogBv1vVlnZCCPw8Z1hpKggSSlJZ1QoVrdAW3Ykm/pBTs0kc3N7XsH1G2wwrGk+yz76yjEdPX8HvvH4vBkOeto+/7/oRJHIC/qNurYoyOacuxDQWdCNXkkxbntwrEl1c92F1qDpptEWX6OKyXoqWuA69RQVVmqZVxA6cmU/hqm2h6nF1U2mq/n0ai0IjRPz2yDRkRZNJFCr95vXTc4ByB7OVlSZanOhRmgB1q1SW0trSa31up2GlqbZ3zvxg/TceGALnIHi4wRSdIJVx5koaB0dCDX4SiPrp/jltRdPnnpjE9j4PPvC6Paoef9tEDBE/v2aK7vJyDjsi7f1MwNpUcDtDlSazvW12ZN9QEDznwOnKImm90GmxbqWBA7W4DlVKU9UOoO34In4eXpezrRlclmW8NJ/CgeFg9TXortKkLD8Pebu3NCRSUZq6aXjXAyuaTIKa9LwN0msjAR5xG4xSdoq0bqXJtebnWxHPFDWlMfvdHDIGlaZUde+c+RfLfh+P145H8fCpjS2684sZlMRyU89RNV1X413b2StpvP6qmGpvjcvpwFuPbMcTZxaRzAkQpDJmE3lNShMA2/uaaFHPlCblPbF/W9DwBB1NwO6mERxQvyuUns+1Ts/RrKZ2AZdXUgUkcgL2D4eqalt3PU3qlp+bSSTghliWqzchVoUVTSZRU5o2XnBiGuP5Nxt6PU3UK6TGDK52hQrFz3PIGWwLVZWmDt2N3X1oG6ZX8jg1u/auvVESeD16ssHyJQnL2ZLmnKD7rh9BSSrjoZPzmEvkIZVlVSZwoC4VfJMoTd2+wFuVgyN9ODWXNKQY0GW93faJRQNuVQotfc8O6DBKK0VTa6WJ5jOtLZq662mKdClugKLFiN9LWNFkEq2KJmWUUn3S7GYjUxDhINjg92qH2vYcHf/VMunh453IGowcSHdQaQKANx7YBqeD4KF1LbrTcyn4eSd2N2mD6UnXna0E7lGjqloOjfRhb8yPb78wUxc3sLWUpiRTmtZwaKQP6YJoaFF5t5f1UqIBN+JqlKZMEWGfC5yOnZOjYZ+KokmZnLt6WxBuzgkf78RqN5WmbPeCLSm1SXNrnw9Y0WQS+ZIyQdSoMIgE3CgIZVNygewIXaGiVeqtFk1tlKZsSZm00WLK9Ls5w7vnaIaU2dNzlLCfx2v2RvDIuhbdqdkkDmwPNQ1CDHk5uJxEU1YTPYmPhLUpTYQQ3Hf9KH5+aRVPn48DAHaq9DSFfTycDsI8TZsMM8zgqz0wggOK2qFG+dSqbNczGvYimRdantdemk9hbMBbvSHr97q662lKd19pqirkFreysKLJJApiayM4sHUDLjNFUZcaQw3W7QIutWQ0UXy8GUbwysWyQ0oToEzRXV7OVY21UlkxiF7TIJ+JQgjRnNVEi6ZRjUUTALz92u0AgC8/exluzoFBlcWr00EQ8fObomjy8064dKgOm5F9Q0HwToehdSqJfAk859CsThslEnAjVRDXxGg0Qu1y8EaoiR04M5/C/m21QY9+H49kl5QmWZYRz2pPOzdKxCb759in3CTyFRWpsdJkj15tp8gUxOqSXC34eQ4O0l5pqqXXqr8zMiNygHqaOqU0AcCbrlFadHSK7mI8g1xJahk8CdQmUdQym8iDcxAMBttHDaxnNOzDzbsHkCtJ2DHg07QKZDOsUjGaBr7Z4DkHrtoWrAaw6iFRWdbbTSMyUNciatOi05MGTqHZas3M4PmShEvxLPYP14qmsL97SlO6qBSN3Vaawj4XCLH+dZIVTSbRzggOWL+C7hS0PacVh4Mg6HG19TTRD5kmpcmEyIFUQYCD1HbZdYIBP49X74ng4ZNKi46awpvFDVAiAbemic2Z1Ty293tV5Vw14r7rlSW+anbO1bMZUsETOYG15tZxcKQPp2ZTun2c3U4Dp6gdoohnSrrTsqtKUxNf09mFNMoy1hRN/T6+a9NzyzrOp2bAOR0Y8PHM07RVyLcxggNbd5VKpigioLOFFfRwbSMHaDGqJXIg4FaUJiPm/HRBKQY7vWT1LYe24dJyDmfm0zg1m4Sbc2A8Fmj5M1G/tpPP7GpO8+Tc2mMcho93YmIoqOnnNkMqeIopTRs4NNKHZF7AtIrk60Z0e1kvhRYKrQr5giAhUxQ1nW/qCftc8PHNs5ro5NyB+qLJ60KiS6P49Hwa6XLRpPxO6wdBs6LJJIqC0gNv5GmivVqrV9CdIlMUEdShNAHq9s/RD7mW8V8fz6EsAwVB+6oRSiovdGxyrp43XbMNDgI8fHIep+aS2D8caju1o6c9p8fPRAl5XHj4w7fhQ28Y1/RzsaAS/GrGrrJewdpzGzlk0Aze7WW9lKiKG9wlncGWlHZZTWfmUwi4uTWfx7CPRyJX6srnZFmH3cEsIn7rB0GzoskkquGWDZQmnnMg5OEs/2boFHo9TYAyCdbWCF4Z/9VixKXHY8TXlCqIXWnLRANu3FJp0Z2eS7VtzQHKXWJekFRNCBZFCQupoubJufXsivo1t2FjATcEyfqBdq1I5oWetJKszL5tAbicRH/RlC91PW4AUDe0Q6e7jLSvRsM+TLdQmvYPB9co2P0+F8qyuqBfo+ixO5hFJMCz6bmtQitPE0DzP6z9ZugUiqdJ30VFldKU1j7J4qv4kIz4mtIFoaMm8HrecmgYr8SzSBdEHGwxOUeJaFilMp8oAICh9pxeNkNWE1OaNuLmnIoZXEfRJMsyVnO9KUR9vBMel6NlV0DPtO56xpooTeWyjDPz6TV+JgBdXaWiR7k3CzusHGNFk0m08jQBNGnW2m+GTlAuy8iWRM0rVCghrxojuPbMFD9vktLUpaLpzZUWHdA8Cbye6h2ziqC+WtyANhO3Gdg9FbwklpEXJFY0NeDQSB9OzmpPBi8IZZTEctfTwAGldRYNuFu252qeH/1FxWjYh3RB3KCwzqzmkSmKG4qmaip4FxTZ5Yxiwu9FhEY0wCNdEFEUrZtpyIomk8gLEnjO0XT6yA6yYyfICRJkGQY9Te3bc1pPYL7K8RgJuEwXhI5mNNUTC7px0+4BuJwEE0OtTeCAtuGDWho4U5q0wtLAm3PNdsUM3i79ej29WNZbTyTQeqKzFnFipD3XOHbgpbr1KfX0d1lp6kVrDqjbm2lhM3jbookQchUh5MW6/6QIIb/fjYOzE0WhDA/X/OWMbtH9c5lKwePXWzR5OWSKIqQWBsi4jqC5qtJkqD0ndq09BwAPvGU/PvbOQ3Bz7f1htZOPOqXJQYBtfdozmoxSLZpsqjTRHWkscmAj1AyutUXXq2W9lFibIYp4poSgm1O92LoRzWIHzsyn4CDAVeumUMNd3D+3bCBOwSj099q6aJJl+awsy9fKsnwtgBsA5AB8u+NHZjPyJQlevvmHKBLgsVrZBL+VyBSVE6De9hydTss0UZv0jv/SIi6rc2mvLMsVT1P3TuxHxvrxyzeOqXps9eSjQt2cXc1jW8jTEzk+6Obg5hw2LpqY0tSMq7YFwTm0m8Gp0tSLyAGg/QSXkWBLSk1p2lg07Yr6N1xLqkpTtvPtOTP+Pr1ENNgKeoXWs+QdAC7Isny5EwdjZwqi1PLOg74ZVrdYiy5TUXL0t+cqq1SamMH1pIEDtUBKvUt7syUJZVlRwqyIx+VEwK1uYnMmke+JnwmoeUhY0bT58Lic2DcU1Fw0JXu0d44SDfJYyTYf71faV8YKun6fC37eiemVte25M1dSG1pzgPL+IqQ7nqZ4pohoj5Sm2GZoz63jPQC+1ugbhJAPEkKOEkKOLi0tGT8ym5EvSS33JMUqHzK7ejf0QhUiI0ZwAE1H0vWOx/oqkQN6PU3UnN5NpUkrUZVZTbOrecNxA0awcyo4fV/2ShWxOodG+nBKoxm8V8t6KRG/G2K5eQyGkgZuTIkhhGBswLdGaUoXlDDQAw2KJqeDIORxdbw9VxQlpApiDz1NdOWYdc8HqosmQggP4G0A/q3R92VZ/oIsyzfKsnxjLBYz6/hsQ0Esw61CabJyBd0JaHtO76oRarRuqjTpHP+tKk06PU3d2DtnlEjA3XaHliiVcSVV6EncAMXOqeBUFWFKU2MOjvZhNSdgNqHeDN5rIzhtTTX77CjtK+PHtj7g8uUraQDA/uHGqfphn6taUHaKlUonpBdp4IC6yIdeo0VpeguAY7IsL3TqYOxMoSTB2yANnKJ2EeRmw2hxQdtfzQIuq+05jT14j8sBQvQrTelKEdet6Tk9RPztlab5ZAFSWe7J5BzF1kVT5X3ZregJu1Ezg6dU/0wiJ8DNOQwZrY1AW1NLDTLOBKmMRE4wRYkZDfswu5qvqnBnmkzOUfoqqeCdhOa69SINHFAUuIi/deRDr9FSNP0KmrTmGGo8TerDBjcT1GitZ2EvoEJpopkpGnvwhBD4eU630kSPx+pKU7uN4VQB6Gl7LuDGSq5kyyGJZF5AwM21XWuzVbl6WxBOB9E0QZfI9SYNnNJKaVoxIQ2cMhr2Il0UqzeEZ+ZT6Pe5sC3UeIo17HN1fGkvNWD3SmkClNffykHQqj7phBA/gLsAfKuzh2Nf2nmagm4OvNNh6amATpApGo0cqBRNLfwFQY++8V8f7zSgNIlrjs+KRAM8VrKt97r1MtiSEgu6Icu1C5KdYGngrfG4nJgYDGgygyd6lAZOqaXpbzxXG907Vw9Vd6crLbqX5tPYvy0EQhpn/YV9fMdzmujfHOtl0eTnLR12q6pokmU5K8tyRJZlfYuEtgDtlCZlSojfckpTuiiC5xzgW2RYtYIqVM12Li0ZCGILuLlqUaeVlB08TX4eZbl1IN5spWga7kFGE8XOWU3JfMnShbMV0GoGT+R6W4iGfTwcpHFchxnBlpRaVlMOUlnG2SaTc5T+LihNy1VPU++UPiUI2rrnAqYpm0S+VG6rdqgx5m42MgVRd9wAoEyNBN1cSyO43rs+n9uJnM7IAap8WdrTVPXRtSiaEjkMBt09848A9k4FV5Qm6xbOVuDQaB+WsyXMJwuqHt+rZb0Uh4NgoElW07KJy2zH6gIuL8azKAhlHNjeomjy8sgUxY62sePpIrwup+7OgBlEKmtstK7f6RasaDKJgtC6PQcokq6VRyk7Qbaof+8cRdk/11gRWs5qTwOn+HhOd7hluiCCdzrg1qmgdQM147szPY4bAGqtAHsqTaw91w66K/H0nDozeK+W9dajnKtbKE0mhD+GvByCbg4zq/k6E3jjyTkACPtpKnjn1KblbKmnKhOgFKRiWW56zu811j3j24yCIMHTYnoOqFXQW4lMUdRtAqcEPS2UJgPtOT+vX2lS0sC5pv4DKxBVEXMx28NgS4q923MC+r0so6kV+yorQc4tpNs+VpZlJHNCT5b11hMNNFaa4pkiPC5HdQ2TEQghGKnEDpyZT4FzEIwPNt8rSbPAOjlB18u9cxTaObCq/5cVTSYgSGWIZVmF0mRt2bETpAuiYalXUZo2Fk1Gx399bg5ZveGWXd47p4faHqfGJ59yWcZcIt/TjCZAMQsH3Zxti6ZeX+CtTsDNYaTfi0kVRVNekFCSyj1tzwHNg2FpsKVZN0ujYV9VaRofDLTcK0n3z3Uyq0nZ49nb154Gh1rVDM6KJhMoCIpa0c4XEg3wKEnlqol4K5ApGvM0AYpvqNFrVvUX6Aya8/NO5HSHWwqWNwD3tzC0AsBiughBknvengPsmQpeFCUUhDJrz6lgYiiAcwuZto+rpoH3+DWNtFCazNzLpgRc5nFmPt3SBA6gqmhudqWJtgfV7M3sBaxoMoE8LZraSLbVN4PNLg5GMMXT5OEaKk1GJ1l8vAGlKS9YXmlyOggG/I29GQCqacS9DLakRG0YcEnXbFi9eLYC+4aCuLCUgdQi/gKoFQS9XksTDbiRK0kbIknimVJ1JZYZjA34kCmKuJIqtPQzAbW1Mp3yNJXLMlYs4Gmy+nWSFU0mUBSUaQZPG1Mwvbi3CxzcTJjhaQp5XQ09TUsGi6aAWzGC62mXpgsigm7rXyyVdN3GJx8abDna4/YcoChNVpXjm8FWqKhnYjCAoljG1LoFtetJ9HjvHKV24V57rjZbiam/YWmnNIUr7fZOZTUl8gKkstxzpWnAx4MQ614nWdFkAlRp8rZTmvzUmKv/4jC9ksOff+cUnjkft4U3Kl0woWjyKHlK60MajQax+dxOlGWgKGof4U0XxOqKFysTDfJNZW4abGmJ9lzAvkoTK5rao9YMbpWiKVa9wa29JzuhxGgpmvy8E5yDINEk6Nco9LrUyzRwAOCcDoR91p00Z0WTCVQ9TS1MfEDNe2MkIv7RU1fwzz+9jPd+8Tnc9Zmf4J+euVTdg2YGRVEy7fkEqYyiWDZFaZJlINNAKgf0B7HVlvZqb9GlCgKCFs5oorRSmmZW8xjw8/DpXKZsJrGgG+miWP0s2QFWNKmHToW1M4P3elkvpRbXUTtXr+ZKpisxdHI1FnS3fV5CCPo7uH+uptz3fhpUzd7MXsGKJhPIl9QpTQO+5vH8allIFeBxOfA3v3QEft6Jj373NG75+BP4yL+fVDXS245PPPIy3vOFZw0/D1C3d86wp6nxKpV4xlgQm6/y76U1dkCUysiVJEsHW1IiTaaAABo30HuVCbBn7AAtmnptWrYDfjeH0bC3rRk8YZGWZy2uo/Z+XDZx7xylz+tC0MO1VZkoYZ8Lq9lOKU3m/316sXIqeO9vMTcBhUp7p11OkyI7ugy9GRbSRWwLefCuG0bxrhtGcXw6gX/+6WX869EZ/MuzU7h59wA+9IZxvG5fTNfzvzSXwvnFDGRZNjxWS1efGFealJ9P5UUgXPv6cqaoe3IOqO3D02oGp6tXrG4EB5QTIFVw1k93zqzmcNVQa/Npt6hPBR8b6G1ulFqY0qSNfUNBFe25ErwuZ08T6gFgwL8xGDaeNuahbMYfv/lq7I74VT22k/vnzFwRY5RowI2XVIahdhumNJkAVZrUfNCjAbeh/XOLqQIGg7U9YUfG+vE3v3wEzz5wB/7kzVdjeiWH3/yno7qTrmdW8yiKZaR1/nw9tLgw7mmqKE2F9UqT/jRwoKY0ZTXGDtCkWjsUTdWspnUtYVm2RkYTxY6p4Gx6ThsTQwG8spSF2GINSK+X9VJodlh9e462r2IGbtQa8Wu37MStE1FVj+3zuarvO7NZzpTgINZQTqMB60aQsKLJBKgPo124JWBcdlxMFzEY2lgoDPh5/M4v7MWf3r0fJalc3ZytBUEqYz6pmIPNuHhlTGrPBVu054wUTVRpWj9W3A5avNnhYhlp0GYAlCKqIJQt054brChNizYrmoJuDk6HdVPhrcS+wSBKUhmXW0zQrfZ4WW890eDarKaqh9LfOyUm7HN1TGlazhYx4HfDYYH3c8TPI10QURSt53FkRZMJqA23BGqp4HpZrzStZ0eltTG9ktf83FeSBdABtcWUiUWTWe25wnojuMGiSacRnLYdbaE0NRmdrk3OWaMVNuBXxoyXUuqWulqBZN76AadWgk7QtTKDJ3u8rLee9WbkeKYIzkF6WtQp7TmhI5PTS+nep4FT6M3eigUDLlnRZAJ5DUqTEdkxUxSRLUkYaqA0UcYqysF0mzyURtSrU2ZIoxmTigvanquf6pMq479Ggub8bp3tOao02cAIHvVvHJ0GgNlK0WQVpYlzOhDxu+2lNFlIFbED44MBEIKWZnArLOulrN8/t5wpIhLge6rE9Pt4lMRy9ZpjJsvZ3qeBU6JNbvasACuaTKBAwy3VtOcMyI6LlbvwRu05ijJC7mwbItcIqj4A5rbnjO6eo0VX/dbrlWwJZdnYtnE6aq+1PUeVJjsUTc1WEtA0cCtkNFEGgzYrmvKsaNKCl3diLOxraQZXPE1WUTvWZpwZ9VCaQSdTwRXl3iqvfW0wxGqwoskEaNXvbpMIDtQu8noq6IVKy2yoRXuOEIKxsK96UdTCzEoODgK4nMScosmk6TnOqWwVrzeCmzHpUVWaNEYOUG+VHdpzPt4Jj8uxwdM0m8gj5OEsVfgNhtxYTNurPceKJm3sGwpgsonSJMsykvmSpZSm1Vypaly3wl622tJe8xWY5Uyp58GWFKY0bXKKggQ351Al20YajKOoIxkAACAASURBVLKqhV5QWilNgLLPSI+naWY1j+E+LwaDHlMuXlWlyYTwxJDXtcYITl8/+nrqwcM5QQiQ0+lpMmpw7waEkErA5UZPk1X8TJTBoNsUL123SOat00qyCxNDQbwSz0BoMEGXLUkQJNkS01uAcuGWZWClUqDE08We72WjKpzZSlOuJCJXknpeFFKaDbBYAVY0mUBekNoGW1IGQ4pKpOfiQH+GPkczxga8mF7NaTYLKhdSr2nLU+neOTM8ACGPq7HSZKA953AQ+FxOzUpTuiDAxzvhctrj4xMN8BtS6GdXrRNsSRkMehDPFNsudbUKTGnSzr6hAARJxuXl7Ibv0aRrqxjBawGXJciyXFnWuznbc8sGtyuYjZ93ws05mq6A6iX2OOtbnIIgtV2hQtnepxQ8dLRfC4tpJQ082KbdNRb2IVeSNL/hpldzGA17TdsDlimI1RaYUYIebo2nyaz0Wp+b0xU5YIfWHEWZ2Kz9e8qyjFkLZTRRBkNulGVYNgm4noIgoSiW2fScRiYG6Q66jS26ahq4RdQ7qnbEM0WkiyJKUrnnSgwtKM1uz1UzqCyiNBFCKpmG1jsXsKLJBPJCWbXSFA244XISzCa0t78WUkUMhTxtk7rHqrED6n1NJbGMK6kCxsI+DIbcpixLpEqTGYS8a5WmpUwRvNOBkMHixc87kdE4PZcuiJbyArVj/SqVZF5ApihaUGmqZDXZoEWXYmnguhgfDMBBGi/upUWTdZSmmq+mmgZucrClVmpKk7lFk9WUJqCxQm4FWNFkAvmSpMoEDigtoW19Ht1K06CKdlQ1q2lV/e+YT+Yhy6gqTcvZUsvkXjVkiiICJhUXIQ9X9RIBQLySKWJ01YvfzenyNNlJaYoE3FjOFqvt2hmLxQ1QYpUBBzukgrMVKvrwuJzYMeBraAany3qt4hOrV5qsEGwJAG7OCR/vxKrJ7TkrrVChRALNl433ElY0mUBRVO9pAoDhPi/mdShNi6liWz8TULsYalGaahdSH2JBN2R545i6VjJFsW0rUS3rlaZ4pmjIz0Tx85zm3XNKe84aJ3Y1RPw8BEmutjerwZb91jOCA7DFBF2CFU26GR9svIOOFgJWMYKHPBx4pwPxTMlSRUW/19UBT5Py9w0YGKwxm/XholZBVdFECOknhHyDEPIyIeQMIeTVnT4wO5EvSaqCLSnb+zyY06E0LaTUKU1+N4eIn9dUNNHHjg14Tds4b6anKeRRpueoWmLW+K/P7UROsxFctJWXhb5O8YpXaDZhVaXJPu25ZI4VTXrZNxTAxXgWJXGtkp2stJys4mkihCAS4BHPFKtFRa/bc4AyQWd2ey6eKSHo4Xq+KLmeaHCtQm4V1CpNfwvgUVmWrwZwBMCZzh2S/SiIGzfIt2K434uFVEHTlFAtDby90gQAowM+TfvnZlbzcDoItoU81cLMcNFUFBFwm9Se83Ioy7VMJbOC2Pw8p3mNSipvLyP4+lUqM6s5+HinZdogFI/LiT6vqyMBl7Is48JS8yRqrbD2nH72DQUhlmVcWjdBl8gpU6lulUM13YAOUSxlSiAEGLCA3yrsN3//nBUyqNazXiG3Cm2LJkJIH4DXAfi/ACDLckmW5USnD8xO6FGaBEnWZLaupoGrbEnt0JjVNLOaw3CfB5zTYZ7SVDTP+1O/tLdclrFsUjqvj9enNNmqaPKvzTyhcQNG/WCdQEkFN78999hLC7jjb36MM/MpU56PFk1WKzztwMRQAMBGM/hqTrBMa46iKE1Ke27Ax4OzQMyIojSZ72mySho4Zb1CbhXUvAN2A1gC8CVCyAuEkC8SQvwdPi5bURDKcLvUf5i2V0a95xLqixp6961WaRoLezGXyKtWs6brcnvom9XIxUuWZXOn52jRVBCQzAsQy7Ip6bV+tzalqSBIKEllW03P0ZMhnUSxYtwARUkFN/8k+fOLKwCA515ZNuX5ktVUePu8D6zC3hidoFur/Clp4Na7cC9nipYItqT0e11VT51ZLGdKPTe5r6fZsvFeo+ZKzwG4HsD/lmX5OgBZAH+6/kGEkA8SQo4SQo4uLS2ZfJjWpiBoU5qG+5QL1nxSfVGyUFGaWi3rrWdswAexLKue0ptZzWGskhDtcTkR8nCGlKaCUIZUlg3vnaOEvLX9czTHx4w7Ix+vhFuq7ZvXlvXaR2kK++nJR3ndaIipFRkMejriaToxkwQAPD9ljkierLRonT1c3mpXPC4ndkb8mGykNFlMuatXmqzSvgpXPE1lE0NglcEaaxSFlGjd9KKVUFM0zQCYkWX5ucr//waUImoNsix/QZblG2VZvjEWi5l5jJanIGjzNG3vV9QiLUoTLWBiLfbO1UMLIDWLe4uihIVUEaN1azViQbehZYnpolJcmLVqhCo76YKApbRy52FGEJvfzUEqyyiK6uIVaOyBnRQGl9OBfp8Ly5kS0hWlbtRiK1Qog5U0ejPNn1JZxqk5pWg6dnnVlOdMsTRwQ0wMBja05xK5kmUymiixgBslqYyL8axliqZ+nwtlGWsiWIwgSmWs5gQLK002K5pkWb4CYJoQclXlS3cAeKmjR2UjZFlW1qhoKJr6vC54XU7NSpPHpT7MkWY1zajwNc1V4g/qp6kGgx5DSlO2EhhpZuQAoCg9ZqxQofgrURFqfU30REWVL7sQ8fNYzhark3NWbc/FgspFKmli++H8Yga5koRDI32YTeR1ZaStJ8GKJkPsGwri0nIORbH2uUvmBctMzlHohXs1J1ioaDI3FXyl0rY343xqJtR0H7dhew4A7gfwFULICQDXAvh45w7JXgiSjLIMeDR4mgghGO73aPY0DQbbp4FThvs9cBComqCjcQP1RVPM4P65DF1qa5qnqdaeMzMzxVc5PrW+ppRNvSyRgBvxTAmzFg22pFR3M5roazoxo7Tk3v/aXQCAY5eNt+jY3jljTAwFIJVlvLKkTNDJsoyEBY3g9ecYq3iawjQV3KQbC1qURC2U0QQAnNOBsM9lubVKqq70siy/WGm9HZZl+R2yLJujcW8C8oJyp6Q132Kk34s5jUqTWj8ToLRkhvu8qrKaaNghXb8CKEWTkQsXbc+Z5Wmqn56LZ4pwOogpJ1g/rxyfZqXJZkVTrGJorQZbWrVo6kBW04mZJAJuDvccHobH5cDzJrToWNFkjH1DdAed0qLLFEWIZdly7bn6lpVV9rKZrTSZqdybjZIKbk+lidGEgs6iabjPg3kdSpMWxga8qjxNM6s5cA6yZjIvFnQjV5I0ZxhRMlXvjzlFE8854HE5lPZcuoSIn4fDBBOurxK+qTYVPF2gSpPN2nMBHsvZEmYTebg5h2UuAOvpRCr4iZkEDo6E4OacODzaj+enzCmarGZathN7Yn44HaS6TsVqy3op9eZoqxilwybvn6NKTsRiShNQ2T9nN08TozW0aNLiaQKUCbqlTHFDKm4zlBUq2i50OwZ8qvbPTa/msb3fu2YSyGjAJS1CzGrPATQVXDR1kqWqNKlc2puya9HkdyORE3ApnsVIvzUzmgDz23MlsYwz82kcGe0HANywM4zTs8nq51Yvybxgq1R4q+HmnNgZ8VWVJqst66UM+HjQj4rVPE1mZTXF09b0NAFMadqU0Paclt1zgDJBJ8u1KIFWZIsiMkVRu9IU9mEpXWx7gZhZzWFsYG27phpwqbPKr3qaTCwu6P45s/bOAUrkAKC0B9SQLohwkFqxZReoH+PETNKyrTlAKbJ9vNO09tzZK2mUpDIO06JpRxhiWa5GEOihIEgoiWXWnjPIvsEgJhcrSpPFlvVSFF+N8tmxStHU53WBEJi2tDeeLYJ3Okwb2jGTqJ8pTZuOgqAoRVqM4EAtq0mNGbwWbKntQ0s9SjNtzOAzq3mMrlveanQPWLrYCaWJQ7ogIp4pmZZeS48vp7I9l8oLCLg5U1qD3YS+XldSBcuawClmpoIfr5jAD4/2AQCu3xkGAEO+JrZCxRz2DQVweTmLgiBZbllvPbRtZZVltk4HQcjjMq89VzmfWlF9fs14FL9845il9s9Zr7S0GfmSPk8TzWpSEztQW6Gi3dMEANMreYwPBhs+piBIWEoXN1xIqedlSefFK1MQwTkI3Jx5dXnIq2QNmdmeq3ma1BvB7TY5B2BNerpV4wYog0GPae25EzMJhH2u6vt7wM9jT9TPiiYLMDEURFkGLixlqst6rZYIDigK05VUwVLLbMM+l3lKU6ZoynaFTvCma7bhTdds6/VhrIEpTQYpiHqN4BWlSUVmzIJBpamVGZxOU42ua8+FfTw4B9HdnssWRQQ8nKl3LyGPC/PJPIpi2TSlqeZpUqk0FURbelnqTZ5WDbakxELG4i7qOTGTxOHR/jXvw+t3hnFsalX33WvVtGzD94GVoBN0kwuZagFgxdd0V9SP3VFrbQ7rq6SCm8Gyicr9VoAVTQYplPQZwf1uDn1eF+YTnVOaYgE3PC5Hy9gB2robW3chdTgIogH9F6+0iXvnKCEvV8sUMenOiP67qVWaUgXBdiZwYJ3SZIf2nAqvXzvyJQnnFtI4UmnNUW7YGcZKtoSL8ayu52VKkznsjvrBOQjOLaSRyCltb95EZdosPnLPfvzT+2/q9WGsIexzmWcEt7DSZEWs9w61GXqVJqASO6BCaVpMF+HmHJpTqAkhGA37WgZcVpWmBuqDkYDLTMH8oqm+LWZW0eRwEPh4p2qlKV0QbbV3jhLycHA5FbXF+p4mD7IG4i4op+eSKMvAoYoJnHKDQV8TK5rMgecc2BX149xCBol8ybKvp9/NVfc3WoWwjzclp0mW5YrSxIomtbCiySD5kmIE16o0AcD2fi9mVShNC6kCBkNuXa2usbAX0y1WqUyv5uBykmrEQD1GAi4znVCaOlA0AYCP5zR4mgTbBVsCSgEd8bvBOYhmxbLb1LKajLXojlcm5NYrTeOxAIIeDsd05jXRoqnfa60LqR3ZNxTA5KKiNFltcs7K9JukNKUKIkqSeXaHrQArmgxSC7fU/lKqVppSRQzpvNDtGPBheiXX1L8xs5rHSL+34TRYzEB7jnqazKReaTMzaM7vdmqanrNjew5QYgfW53FZEZpHZrRFd2ImgW0hTzX7ieJwEFy/I2xIaSLEflldVmRiMIiplRzmkwXLZTRZmX4vj0xRhCCpy/lrBh3nt8qKGDvAiiaD6F2jAihKUyInVCfwmrGQLmgOtqSMDfiQLopNF6DOrObXrE+pJxZ0YzlbglTWbpjtiKepovAQUlvmaAZ+nlPVCpJlGZmiPY3gAHDz7ghunYj2+jDaQpUwo0rTyZlkNWpgPTfuDOPcQkbXYuBUXkDQhrETVmTfUBCyDJy9krJcGriVCftpKrgxtYnu/tsZsZbR3cqwoskgBUECIdA1Wk9jB9pN0C2ltK9QoVCvUrMW3exqrqnHZTDkhlSWdfXOO+FposXKgI8H5zTvret3O5FVkQieLUkoy/ZVGP78rQfw8Xce6vVhtMWM9lwyL+CVeLZp0UR9TS/oaNEl8wK7wJvEvqEAAKAs19aDMNpTSwU35muiiewTgwHDx7RVYEWTQQqCBA/n1OU3orEDrSbociUR6aJoQGmqZDU1MIPnSkpQZLMRdJrVpCfgsjOeJuX5zDYt+nhOVXsulacrVNjJvZP0+1zgnQ5DAZenZhU/0+F1JnDKkbF+OAhwTEeLji3rNY9dUX91QIF5xNRDQ0ATOpTSeiYX0tje52HnNA2woskgeUHSvEKFsl1FKjgtWPR6mlplNc1WJ+caK016V6lIZRm5kmS6p4l+sM3uv/vdTlVG8HRlNYwdjeB2ghCiTG4aWKWyPgl8PX43h/3DIV3LexM560562Q2X01HNQGJGcPVQ/9dq1qjSlMHEUOPgY0ZjtmTR9KGvHMN3Xpw15bkKQhkendkiQ31KUdKqPUd30w2F9BVNIY8L/T5Xw6ymVnEDQF3RpLFN0ollvUDNCN4RpUmFp8muy3rtiJHJTUDxM+2M+FomTN+wM4wXpxIQNZppmdJkLvSibcU0cKtCC0wjniapLOP8UqbaImWoY8sVTbmSiIdOzuOTj57VfLJsRF6Q4NGpNLk5J2JBd8v2HL1w6G3PAUpw5fTqxsJsuhps2UZp0njxqi7r7ZAR3Oyiyc+rU5roXR27YHYeo/vnTswkcWikscpEuWFnGNmShLMVX4daknkRfayVZBr7KiuerLh3zqrQ3CgjWU2Xl7MoieVqMjtDHVuuaKKtsNlEHo+evmL4+YoVT5Netvd51ClNBrJ1xga8mGmiNPGco2kR4uM5BNyc9qKJLus1WZHxuJz4T6/eibccMncXkc+tztP0SiVBepfFVipsRgZD+pWmeKaI2UQeR5r4mSjX71DM4Fp8TbIsI8WUJlPZP6xctGMNsuIYjfHzTnAOYsjTdG4hAwCsaNLIliuaaJgk73Tg75+8aHh7shFPE6CYwVst7V1KF8HrSAOvZ2zAh5nVPMrrogNmKpNzrUanYzru+KtFk8lKEwD85dsP4lW7Bkx9zoCbgyDJKIqt1abzixnEgm52wewCg0EPEjmh7b9JI0608TNRRsNeDAbdmvKaCkIZJanM3gMmcuf+IXzp/a9q++/FqEEIQb/B/XOTFYV1nE3OaWLLFU1UafrPt+7G8emE7lRgSkEo6wq2pAz3ezCfyDct3hZSBQzpTAOnjIV9KEllLKwrfqZX8m2Xt+oJuKTtObt4f3yVojfXJnbg/GIG4zF2gukGgzpbw4DSmiMEuKZNe44Qght2hjWZwdkKFfNxOAjecNWgqcu9twJhnwurWQNK02IGo2Ev/B24ud3MbLmiaXY1D6eD4HffsBd9Xhe++ORFQ8+XL0m6VqhQtvd5kS1JSOUbt4cWDGQ0UegE3fqsppkWGU2UWNCteXqOKk12+TD6+f+/vTsPjvwu7zz+edSt+5xDx5ye8djYjInPiWNjQ8AG24CDNwnskoQsIcl6d0MSJ8UuBamtpLK72crWphKyGzZVLgNJFWwIMWbDEoJxwOwyJBjG9uBrzFgzPubwSJpD14xaarWe/aMPaTQ6vn3p92v1+1U1ZamnkR79GKk/er7f3/PN1nl+hSU6d9eRkUnt6WNpbi0UpoKXGJqu6O0I6nTedNkGHTs7FTx9nNCEuCj3/LnDpyZ0FUtzRau70HRydEoDXS3qamnUz//ETj36/Cm9dmb5A21Xk0pn1FxOaOrJjR1YZl/T8ES201SO/EbvhXfQTU7P6tyFdFhoislG8Gppa851mlbYDD4yMa2J1CydpjVSmApe5NgBd9czx0eXnc+02I25IZehHef8cgihCVHrbmssaaK9JKUzczp6mnEDpai70HRiNHvWmiR96NZdajDTZ/+x9G5TKl1ep2lLbir4cmfQDVeg07RtQ6vMLh5wmZ/RtGO15bnOZk2kZgtn7IXId5o6m2vjhaXQaVph7MDgcHbT5BV9/JBZC/PLc8Xtpzs5ltLpyRldtyNsf8w1W7vUlGwI3tdEpwlxsaGtseRO06tnziudccYNlKAuQ1P++JKB7hb91HVb9cUfHCvM4CnWVDpT1p6m+QGXl744lDsNPK85mdBAV8tFAy7zXaeQTpNU3N6S+eW50sPkWirsaVqh0zQ4kg9N/JBZC5s6mtVgxS/PPZvbBL7auIG85mRC127rJjSh5mSX59Il3czEnXOlC3q1N7NXzOxZMztoZgeqXVS1ZOZcp8ZShSUxSfqV23fr/ExGf/39YyV9zFR6rqxOU29ns5INtmSnKb80UW6nScp2lI4v2NN0/Fw+NK3eaZKKe/GanJ5VS2NDRc+Hq6b83qvVOk0dzcmyl0oRJtFg2tTRXPTy3A+PjynZYHrjlq7g/81Nl23QcyfGg7qphdDE9GpErLutUTOzc0qli583+KNTEzKT9rDdoGjFvKq93d2vd/d9VaumykYmpjU75xeFpjdt69Ytl2/UZ7/7ctHDLt0912kqPTQlGkz9XS1LdpryQaUSL9TbN7ZetDx3/NyUWhobtHmVI0ny588V02maSM2qo0aW5qQFoWmFjeCDw5Pa09fBHT5rqJQBl88cH9XVWzqL+p688bINmsnM6fmTY6s+d3wqLTOps0b262H9KhylUsIS3UvDE7psY1tZ43LqVW20AirkxGg2NGxbtCT1q7dfrpNjKf39c8UNu5yezYasckKTJG3pblny/Ln8YMtKdJp2bmzTqfFUYe7NsXMXtH1D26ohIL80WMwddOenZ2tm3ICUHRQnSedXGDnAuIG111fkUSpzc65njo8FbwLPuym3GTxkiW5sKq2ulsYVZ5sBa2FDrttZSmjizLnShYYml/QNM3vSzO6vZkHVlB9sua3n4tB0x9V92r25XQ9952hR68P5dn45y3NS9g66pQZcVrLTtGNDm9znN4AfPze16n4mSdrUnt1bMhJ4S7aUXZ6rlTvnpOxEcEnLTgUfT6U1PDHNfqY11tfZUlRoevXsBU2kZnVt4H6mvM0dzdq1qS04NLGfCXGQP6uv2PPnpmczeuX0eTaBlyg0NN3u7jdKepekj5jZWxc/wczuN7MDZnZgZGSkokVWSr6bs3VRaGposOywy+NjRU8HlirQaepp0amx1CUTu4fHU2pKNlTkh3RhVlORoSnRYNrYXtyspsnUbM1sApfmQ+9ynaYjuTvn9vQyo2kt9XU168zktDJzYb/IzE8CL67TJGWX6J58dXTVX5oITYiLUg/tffn0ec3OOZvASxQUmtz9RO6/w5K+LOnmJZ7zoLvvc/d9vb29la2yQk6cm1J3a+OSXZCfvXGbetqKG3Y5le80NZW3yrm1u1UzmTmdOX9xm3V4Ylp9neVNA8/bsXF+VtN4Kq2xqfSq4wbyip3VNDFdW3uaEg2m1sbEsp2m+XED/Ga2lvq6WjTn0pnAwP7DY2NqaWwo6TfofZdt1OnJaR3J3SW5nFFCE2Ki1D1N+TvnrmR8SklWfbU3s3Yz68y/LekuSc9Vu7BqODk6dUmXKa+tKalf+ImdevSFU3r1zPmgj5dfnivnwF4pu6cpX99CQ+OpwryacvV3tqgp0aBj5y4U7qJb7c65vL4iQ1Ot7WmSsuMRzi8zcmBwZFJNiQbt3Bh2vVAZfUXeufnM8VFds7W7pLs233519he9b7wwtOLz6DQhLuY7TcWFppeGJpRoMF1O57wkIT9d+iXtN7MfSvq+pL9z969Xt6zqyA62XH5T9b+8dZeSDabPfveVoI+X7zS1lHkHQj7ILR47MDwxrf6u8jeBS9klyO0bWnX87NSCcQOrL89J+UN7ixs5UEt7mqRsaL6wzMiBI8OT2rW5rWZGKKwX86Fp9f10s5k5PX9yPHg+02Jbult17fZufeP5lUPT+FRaXYQmxEBzMqG2poTOFbk8d3hoQpdtait7W0m9WvVVwN2Puvt1uT/XuPsfrEVh1bBwGvhS+rta9FPXbtUXDxwLGk+fmqlMp6lwlMqisQPZw3orE5okafvGNr129oKOn8t3msJD0+nJ6Uv2XC0nu6eptkJTe3NSk8vsaRocnmRpLgJ9XeFHqQyOTGoqnQmeBL6Uu/b26+Cx0WXPoXN3jU2lC7/hA1HraW0sek/TS0OTegNLcyWrm1+dx1NpTaRml12ey/vl23frwkxGXzl4YtWPmZrN72kqLzRtaGtUc7Lhok7T1ExGE6nZwnDJStixITur6di5C2prSmhj+8ozmvJ6O5qVznhQkJyezWgmM1d7y3NNS+9pSqUzeu3sBcYNRCA/Iyyky/m9I2ckSTft3Fjy57vrmgFJ0mOHlu42TaUzSmec5TnERk9bU1HLc6l0Rq+c4c65ctRNaHo918VZLTRds7VLbU0JvXx69UN8p2byd8+VdxnNTFt7WnVywdiB/JJEJTtNOze2afRCWi++PqHtG1qDN5gXjlIJ2JCbvwOt5pbnmpNL7ml65cx5zbm0h07TmmtKNmhDW2PQ8tz+wdPaubFNOzeVvu/syr4OXbapbdklOo5QQdxsaC/u/LkjI5Oac+kNA3SaSlU3oWm5wZaLmZkGuloKgyVXUqk5TdKlAy6HCkeoVLDTlNvI/ORr54I3gS+sIWQz+GQq262ptdDU3pRYck/TkeHsTQEsz0Wjr7Nl1eW5dGZO3zt6Vrdfubmsz2Vmumtvv/7xyGlNLHEWJaEJcdPT2qTRgBWAvJc4c65sdRSalh5suZT+rhadCghNhY3gFQlNrYVumFSdTlN+xMDM7Jx2BO5nkhaeP7f6NZmYzn4D19qepram5JIH9g4OT8pMunwzoSkKfV2r34Tww2Ojmpye1VuuKC80SdklunTG9X8PXzprbuwCoQnx0tNW3J6mw0MTSjaYdm3izrlS1U1oOjk6pcaEFfZJrGSgOztscjWpCoambT0tGp5IFc6/q06naT4oFdNp6i2h01Rze5qaE0uePTc4MqltPa2c0RSRkBlh+wdPy0y6dc+msj/fjTs3aFN705JLdKN0mhAzG3J7mkJv0jk8NKHdm9vVlKybl/6Kq5srd3J0Slu6W4POjMr+dnvphO7FKro819OqOZeGci8QwxMpNSUaKnqnTndrYyHMhN45J2WX2loaG4JCUz541NryXHbkwNKdJpbmotPX2aKRiekVJ3Xvf+m0rt3WXThWohyJBtOdb+zT4y8Oa2b24gO8WZ5D3PS0NWrOs4ekhzg8NMnSXJnqJjSdODelrSvMaFpooKtF6Yzr7Cob7FLpOTWY1Jgof2J3fsDl67l9TcPj0+qt0DTwPDMrLNEV02kys8KL12ry37wdtdZpakpoJjN30QtlZs51dISDeqPU19msmczcsksQE6m0nj42qtsqsDSXd9feAU1Mz+qJl89c9Ph4PjQxcgAxUTh/bmr1zeBTMxkdO3eB0FSmuglNK00DX2wgt49otSW6qXRGrY2JigSbfG0n8qFpIlWRg3oXyy/RLVyqCxE64HJyujY7Te1LHNp74tyUpmfnJ1StKAAAIABJREFU6DRFqK9r5bEDTxw9q8ycl70JfKHbr9ys1sbEJUt0Y1NpNZjU0VRb/7axfm3IBfiQAZeDw5NyF+MGylQXoWk2M6dT4yltDwxN/bmuz2obn1PpTMWmqhY6TbmgNjQ+rb7Oym0Cz3vT1m5t62kteomhtyPsKJWavXsud8DwwrEDgyMTkrhzLkr574Hlvhf3D55WS2ODbrpsQ8U+Z0tjQm99w2Y99sLQRUv0Y7lp4CFL/MBa6Cni/LnDQ9mfZ1fSaSpLXYSmU+MpzfnqM5ry5jtNK4eEqQqGps6W7H6j+eW56nSa/u3b9ujR335r0d2x3s7mwDlNszKT2mps43RbrnuwcOwAB/VGr3CUyjJjB/YPntbNuzepucyp/IvdtXdAp8ZTevbEWOExzp1D3Gwo4vy5w0MTako0aFcZs8xQJ6HpZOBgy7zsXiKtOnYg22mq3CXc2p0dcJlKZzSemi0cI1FJyURDSV2g3s5mjV5Ia3p26aNG8iZy585Vci/WWliq03Rk+Lw2dzRVZIMxSrPS8typsZQGhyd1+xXl3zW32B1X9ynRYHpswQG+hCbETaHTdH715bnDQxO6vLedMzTLVBdXLz80crXBlnmNiQZt7mjW0Cp7mlLpuYreir6lp0Wvj00Vfquu5LiBcuVrOTO58m80k6naO6xXWqbTNDKpy9kEHqm2pqQ6mpNLLs/tHzwtSbr9it6Kf94N7U368V0b9I0XThUeG71AaEK89LQ2qq+zWf/nmZOr3u3NnXOVURehKb+5emt3+ObngYABl1MzmbIP610oP+ByKPcCUY1OU6nmB1yuvEQ3OV2boak9F5rynSZ3Z9xATPQtcxPC/pdGtKm9SVdX6UiIu/YO6PDQpF4+nZ0KP57b0wTERUOD6WP3XK2nXxvVl59e/rzUyelZnRidYhN4BdRNaNrY3lRUV6g/4CiV1Gymop2mrd0tOnN+Rq+duZCrIT6dptABl5PTszU3bkCS2vLLc7lO0+nJGY1NpRk3EAO9nc0aWbSnyd21f/CMbrtic9U2Zr9zb78k6bFct4nlOcTRz9ywTdft6NEffv3Fwt3Li73EJvCKqYvQdHJ0Kuj4lIUGupuDOk2V3IC6JVfjM8dHJakqd8+VKiQ0Tc9m9NLQpDa1xyfshcp3x/LDOdkEHh99XS2XLM/9aGhCpyendXsF5zMttmNjm/Zu6dJjLwzJ3TU2lVYPoQkx09Bg+v33XqORiWn9j2+9tORzOHOucuoiNBUz2DJvoKtFoxfShanfS5mereyepnyNB4+NqjFhhTsj4iAfhFYKTZ//3ms6NZ7SL7151xpVVTn5u/3yU8EHRwhNcbHU8tz+l7L7mW6r4Hympbxzb78OvHpOx85OaXbO6TQhlq7f0aP337Rdn9n/so7mfnYtdHhoQs3JBu3cyJ1z5Vr3ocndixpsmZc/KHelJbqpmYxaK3z3nCS98Pq4+jpbYnUHWlOyQRvbmzQyufT1mEil9WePD+q2KzZVdNDgWmlrurjTdGR4Uu1NicL8LESnr7NZF2YyFy097B88rcs3txfdQS7WXdf0y1165OnjkjhCBfH17++5Ss3JhP7z3x265O8O5/ZnJpgxVrZ1H5rGp2Z1fiZT9A/X/oCp4KnZys1pkrIHBUtSOuOFW63jpLejedl5OQ9952WdPT+jj9199RpXVRmJBlNLY4Mu5DaCHxmZ1J6+jlgF13pVGDuQ+wVmZnZOTxw9uybhfO+WLm3radXDTxKaEG99nS164M4r9a0Xh/WtFy+eZn/41ARLcxWy7kPT8dHspuri9zTlQtOqnabKhaaWxoQ2tWfnbvTHaD9T3nIDLk9PTuuh7xzVu39sQNft6Imgsspob0oWNoIPDnPmXFzMTwXP/tt76rVzmkpnKnre3HLMTO/c26/j57J34BKaEGcfevMuXd7brv/01UOFczTHptI6NZ7Sldw5VxHrPjQVO9gyL99pWq6zMjfnmp6dU3MFQ5OUndUkKZ6dps6lj1L51OODSs3O6aN3XRVBVZXT1pwoLAO9PpbSHvYzxULfonEX3x08rQaTbt1T+aGWS7nrmv7C24wcQJw1JRv0u/fu1cunz+uz331ZkjQ4nL1z7g19dJoqoQ5CU3GDLfO6WpJqbUws22mazqX4SnaapPl9Tf0xmtGU15cLTe7zQ9SOnb2gz3/vNb3/pu3aU+OdmfampCanZ3Ukd+dcrX8960Wh05T7XvzOS6d13Y4edbWsTYC5edfGQoeJThPi7m1X9ekdb+zTf//mSxoeT+lw7s65q6o0z6zerPvQdGJ0Sk3JhsKyVygz00D38gMup3J31VXyGBVpviPWG6Np4Hm9nc2anp3TeGp+Q+6f/MNhyaQH3nFlhJVVRntzUhdmZhk3EDNdrUk1JRs0MjGtsam0njk+qreswdJcXjLRoDuv7pMkdcfojlZgOf/hPXuVzrj+8Osv6vDQhFobE1W/aaJe1N4UwiKdyM1oKmVDb3/X8kep5EcRVLrTlL9bK46dpoWzmrpbG/WjUxP68tMn9K/ecrm2FDFtPa7amhKaSM1qcGRSyQbTZRxsGQtmVhg78E9HzmjOtSb7mRb6yB1X6Ir+jjXrbgHl2LW5Xb/6lt36n98+oq3dLbqyv6NqQ2DrzbrvNGXHDZQWQFY6SmW+01TZ0JQ/6yyO8zR6Oy6e1fTfHv2ROpqT+rW37YmyrIppb5rvNO3a3K5GDraMjWxoSmn/4IjamhK6YeeGNf38e3o79Gtvu2JNPydQjo+8/Qr1dzXr5FhKV7KfqWKCXxXMLGFmT5vZV6tZUKWdOFf8NPC8/u4WDY9fvIcnL1Wl0HTn1X36+m+9Rbs3t1f041ZCodM0Oa0nXz2rfzg0pH/zk3sKJ23XurbmhM5PZ3SEO+dip68z+7343cEz+ondG9WUJNACK2lvTuoT73qjJOmqAX6eVUoxy3MPSDokqatKtVTc9GxGwxPTRd85lzfQ1aKZzJzOnp/Rpo6L9xilqrSnqaHBdPVAPC/xwg25n/unV7W5o1kfvm1XtEVVUHtTUmNTaU2lM3r3j22Juhws0NfVrG++OKR0xvXBWy6LuhygJtx3/Va5XG+/qi/qUtaNoFd8M9su6T2SHqpuOZU1NJZdRionNElLz2pKpatz91ycdbUm1ZRo0CNPndD3XzmrB+68ojBJez1oa05ocnpWmTlnE3jM9HU2K53JdnzfUoMT54EomJl++obt62Y1IA5C2ySflPQxSXPLPcHM7jezA2Z2YGRkpCLFrSQzd+mS2WL5wZbby1iek5Y+SmUqNzm6kmfPxZ2ZqbezWS+8Pq6dG9v0L358Z9QlVVT7ggDIuIF4yXc5+zqbdSWBFkBEVg1NZnavpGF3f3Kl57n7g+6+z9339fb2VqzAJT6PfvkvfqDf/dvnVn1uqYMt8wqdprFLBzqmZquzpynuNuf2NX30rjesu30l7c0LQlNf/PaU1bPe3LDX26/YzNE2ACIT8qp3m6T3mtkrkr4g6Q4z+1xVq1qBmam7tVFf+eHJwr6i5eQHWw6UeOhqb2ezzFbpNNVZaNq7pVM37OzRT127NepSKq491zXc1tO6rpYd14P83aQ/eVX1fiEDgNWsGprc/RPuvt3dd0n6gKRvufsHq17ZCt5/03ZNpGb16POnVnzeiXNT6u1sLrkb1Jho0Kb25iVDUz6wNVd4I3jc/Zef/jH9zb++dV3O/GjLdZo4PiV+9vR26Ku/cbvee936C+sAakdNvuLfcvkmbd/Qqr85cHzF550cmyp5aS5voLuZjeALmJmS63R+Ub7TxLiBeHrTtm6W5gBEqqhXP3f/trvfW61iQjU0mN5303Z998hpncgtwS0lOw28vMnaA10tOrXEVPBqDbdEdPJLctw5BwBYSs22DH72xu1yl7705NLdJnfXydHSB1vm9Xe1LLs8l2wwpkavI2/o79ANO3t0+xof0QEAqA01+4q/Y2Ob3rxnkx5+8rjmlhg/cPb8jFLpufKX57padO5C+pJN51PpDF2mdWZTR7O+/Gu3aSdnzgEAllCzoUmS3r9vu147e0Hff+XsJX9X7riBvPyspuHxi8cOpNJzhCYAAOpITYeme67Zos7m5JIbwk/kBluWuzy33FTwVDpT8SNUAABAfNX0q35rU0L3XrdFX3v2dU1Oz170dydynaayQ1P38qGp3u6cAwCgntV0aJKk9920Q1PpjP7umZMXPX5ydEqtjQn1tDWW9fH7c52moUV30E2lM3V1hAoAAPWu5kPTjTt7dHlv+yVLdCfOTWnbhtay57p0tSTV2phYenkuSWgCAKBe1HxoMjO9/6YdOvDqOR0dmSw8XonBlvmPP9B96diBqfScWug0AQBQN2o+NEnSz9y4TQ0mPbxgZtPJCgy2zOvvuvQoldRMRi3r7MBaAACwvHXxqt/f1aK3XdWnR546ocycK5XO6PTkTNmbwPMGulouXZ6bZU8TAAD1ZF2EJil7iO+p8ZS+89KITuaOVqnE8pyUnwo+Lff5IZpTM+xpAgCgnqyb0HTnG/u1oa1Rf/Pk8YoNtszr72rRzOyczl1IFx5LcfccAAB1Zd2EpqZkg+67fpsee35IL7w+Jqn8GU15hVlNC8YOpNJzama4JQAAdWNdveq/f992zWTm9Jn9r8hsPuyUqzCrKbevKTPnmsnMMdwSAIA6sq5C0zVbu7V3S5dOjafU39mixkRlvrzFU8Hzh/dy9hwAAPVjXYUmKdttkqRtGyqzNCdJfZ3NMptfnsuHJjpNAADUj3UXmu67fpsaE1axTeCS1Jho0Kb2+VlNU4QmAADqTjLqAiptY3uTPvXzN2rnpraKftyB7vnQlO80sREcAID6se5CkyTddc1AxT/mQFeLTozmQ9OcJDpNAADUE1olgbIDLi9enmMjOAAA9YPQFGigq0Vnz89oejYzvxGc4ZYAANQNQlOg/tzYgeHxaU3N5DpNHKMCAEDdIDQFyg+4PDWeUmo2t6epicsHAEC9WPVV38xazOz7ZvZDM3vezH5/LQqLm4Gu+aNUUrlOUzOdJgAA6kbI3XPTku5w90kza5S038z+3t2/V+XaYmVgwVEqTcls1mRPEwAA9WPVTpNnTebebcz98apWFUNdrUm1NDbo1Fhqfk8Td88BAFA3gjblmFnCzA5KGpb0mLs/scRz7jezA2Z2YGRkpNJ1Rs7MNNDVkt3TlJvT1JJkTxMAAPUi6FXf3TPufr2k7ZJuNrM3LfGcB919n7vv6+3trXSdsZCf1TSVzqgp0aBkhQ4EBgAA8VfUq767j0p6XNI91Skn3ga6WzQ0Pq1UOsMRKgAA1JmQu+d6zawn93arpHdKerHahcVRfnluaibDESoAANSZkLvntkj6SzNLKBuyvujuX61uWfHU39Wimdk5nRpPsQkcAIA6s2pocvdnJN2wBrXE3kBuKvgrZ87TaQIAoM6wMacI+angx89NqYU9TQAA1BVe+YuQ7zRl5pzlOQAA6gyhqQh9nc2FtwlNAADUF0JTERoTDdrc0SRJ7GkCAKDOEJqKlN/XxJ4mAADqC6/8Rcof3MthvQAA1BdCU5H6u/OdJkITAAD1hNBUpIEuQhMAAPWI0FSkwvIcoQkAgLpCaCrS/PIclw4AgHrCK3+R6DQBAFCfCE1F2r25Xfdeu0W3XL4p6lIAAMAaWvXAXlysKdmgP/v5G6MuAwAArDE6TQAAAAEITQAAAAEITQAAAAEITQAAAAEITQAAAAEITQAAAAEITQAAAAEITQAAAAEITQAAAAHM3Sv/Qc1GJL1a8Q98sc2STlf5c6xXXLvSce3Kw/UrHdeudFy78tTD9bvM3XtXe1JVQtNaMLMD7r4v6jpqEdeudFy78nD9Sse1Kx3Xrjxcv3kszwEAAAQgNAEAAASo5dD0YNQF1DCuXem4duXh+pWOa1c6rl15uH45NbunCQAAYC3VcqcJAABgzRCaAAAAAtRcaDKze8zsR2Y2aGYfj7qeuDOzz5jZsJk9t+CxjWb2mJm9lPvvhihrjCsz22Fmj5vZC2b2vJk9kHuc67cKM2sxs++b2Q9z1+73c4/vNrMnct+/f21mTVHXGldmljCzp83sq7n3uXaBzOwVM3vWzA6a2YHcY3zfBjCzHjN72MxeNLNDZnYr125eTYUmM0tI+pSkd0naK+nnzGxvtFXF3l9IumfRYx+X9E13v1LSN3Pv41Kzkj7q7nsl3SLpI7l/b1y/1U1LusPdr5N0vaR7zOwWSf9V0p+4+xWSzkn6lQhrjLsHJB1a8D7Xrjhvd/frF8wX4vs2zJ9K+rq7Xy3pOmX/DXLtcmoqNEm6WdKgux919xlJX5B0X8Q1xZq7/z9JZxc9fJ+kv8y9/ZeS/tmaFlUj3P11d38q9/aEsj88tonrtyrPmsy925j745LukPRw7nGu3TLMbLuk90h6KPe+iWtXLr5vV2Fm3ZLeKunTkuTuM+4+Kq5dQa2Fpm2Sji14/3juMRSn391fz719SlJ/lMXUAjPbJekGSU+I6xckt7x0UNKwpMckHZE06u6zuafw/bu8T0r6mKS53PubxLUrhkv6hpk9aWb35x7j+3Z1uyWNSPpsbmn4ITNrF9euoNZCEyrMszMnmDuxAjPrkPQlSb/l7uML/47rtzx3z7j79ZK2K9slvjrikmqCmd0radjdn4y6lhp2u7vfqOxWjo+Y2VsX/iXft8tKSrpR0p+7+w2SzmvRUly9X7taC00nJO1Y8P723GMozpCZbZGk3H+HI64ntsysUdnA9Hl3fyT3MNevCLn2/uOSbpXUY2bJ3F/x/bu02yS918xeUXYLwh3K7jPh2gVy9xO5/w5L+rKyoZ3v29Udl3Tc3Z/Ivf+wsiGKa5dTa6HpB5KuzN1F0iTpA5K+EnFNtegrkj6Ue/tDkv42wlpiK7eP5NOSDrn7Hy/4K67fKsys18x6cm+3SnqnsnvCHpf0vtzTuHZLcPdPuPt2d9+l7M+4b7n7L4hrF8TM2s2sM/+2pLskPSe+b1fl7qckHTOzq3IP3SnpBXHtCmpuIriZvVvZ9f6EpM+4+x9EXFKsmdlfSXqbpM2ShiT9nqT/LemLknZKelXSP3f3xZvF656Z3S7pO5Ke1fzekt9Rdl8T128FZnatshtGE8r+cvZFd/+PZna5st2TjZKelvRBd5+OrtJ4M7O3Sfp37n4v1y5M7jp9OfduUtL/cvc/MLNN4vt2VWZ2vbI3IDRJOirpw8p9D4trV3uhCQAAIAq1tjwHAAAQCUITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAgGTUBQDl2Lwx4bt2NEZdBoAKe+VYWqfPZizqOoCFCE2oabt2NOr7j+6IugwAFXbz3ceiLgG4BMtzAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNiBUzu8fMfmRmg2b28ajrAQAgj9CE2DCzhKRPSXqXpL2Sfs7M9kZbFQAAWYQmxMnNkgbd/ai7z0j6gqT7Iq4JAABJhCbEyzZJC89OOJ577CJmdr+ZHTCzAyNnMmtWHACgvhGaUHPc/UF33+fu+3o3JaIuBwBQJwhNiJMTkhaevrs99xgAAJEjNCFOfiDpSjPbbWZNkj4g6SsR1wQAgCQpGXUBQJ67z5rZr0t6VFJC0mfc/fmIywIAQBKhCTHj7l+T9LWo6wAAYDGW5wAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIQmgAAAAIkoy4AAFB77t56fVU//mE/U9WPD5SCThMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhNiw8x2mNnjZvaCmT1vZg9EXRMAAHlMBEeczEr6qLs/ZWadkp40s8fc/YWoCwMAgE4TYsPdX3f3p3JvT0g6JGlbtFUBAJBFaEIsmdkuSTdIemKJv7vfzA6Y2YGRM5m1Lg0AUKcITYgdM+uQ9CVJv+Xu44v/3t0fdPd97r6vd1Ni7QsEANQlQhNixcwalQ1Mn3f3R6KuBwCAPEITYsPMTNKnJR1y9z+Ouh4AABYiNCFObpP0i5LuMLODuT/vjrooAAAkRg4gRtx9vySLug4AAJZCpwkAACAAoQkAACAAoQkAACAAoQkAACAAoQkAACAAoQkAACAAIwcAAEV79OTBqn78m+++UNWPD5SCThMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAEAAQhMAAECAZNQFAKi+u7deX/XP8ejJg1X/HAAQJTpNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNAAAAAQhNiB0zS5jZ02b21ahrAQAgj9CEOHpA0qGoiwAAYCFCE2LFzLZLeo+kh6KuBQCAhQhNiJtPSvqYpLmoCwEAYCFCE2LDzO6VNOzuT67yvPvN7ICZHRg5k1mj6gAA9Y7QhDi5TdJ7zewVSV+QdIeZfW7xk9z9QXff5+77ejcl1rpGAECdIjQhNtz9E+6+3d13SfqApG+5+wcjLgsAAEmEJgAAgCDJqAsAluLu35b07YjLAACggE4TAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAOY0AXXg0ZMHoy4BAGoenSYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAhCYAAIAAyagLAADUnru3Xl/Vj3/Yz1T14wOloNMEAAAQgNAEAAAQgNAEAAAQgNAEAAAQgNAEAAAQgNAEAAAQgNAEAAAQgNAEAAAQgNCEWDGzHjN72MxeNLNDZnZr1DUBACAxERzx86eSvu7u7zOzJkltURcEAIBEaEKMmFm3pLdK+iVJcvcZSTNR1gQAQB7Lc4iT3ZJGJH3WzJ42s4fMrH3xk8zsfjM7YGYHRs5k1r5KAEBdIjQhTpKSbpT05+5+g6Tzkj6++Enu/qC773P3fb2bEmtdIwCgThGaECfHJR139ydy7z+sbIgCACByhCbEhrufknTMzK7KPXSnpBciLAkAgAI2giNufkPS53N3zh2V9OGI6wEAQBKhCTHj7gcl7Yu6DgAAFmN5DgAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIABzmgAARXv05MGqfvyb775Q1Y8PlIJOEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQABCEwAAQIBk1AUAQJzcvfX6qn+OR08erPrnqLZqX6fDfqaqHx8oBZ0mAACAAIQmAACAAIQmAACAAIQmAACAAIQmAACAAIQmAACAAIQmAACAAIQmAACAAIQmxIqZ/baZPW9mz5nZX5lZS9Q1AQAgEZoQI2a2TdJvStrn7m+SlJD0gWirAgAgi9CEuElKajWzpKQ2SScjrgcAAEmEJsSIu5+Q9EeSXpP0uqQxd/9GtFUBAJBFaEJsmNkGSfdJ2i1pq6R2M/vgEs+738wOmNmBkTOZtS4TAFCnCE2Ik3dIetndR9w9LekRSW9e/CR3f9Dd97n7vt5NiTUvEgBQnwhNiJPXJN1iZm1mZpLulHQo4poAAJBEaEKMuPsTkh6W9JSkZ5X99/lgpEUBAJCTjLoAYCF3/z1Jvxd1HQAALEanCQAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIAChCQAAIABzmgBUxN1br6/653j05MF18TnWg2pfp5vvvlDVjw+Ugk4TAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAEITAABAAHP3qGsASmZmI5JeLeJ/slnS6SqVs5bWw9exHr4GaX18HXH8Gi5z996oiwAWIjShrpjZAXffF3Ud5VoPX8d6+Bqk9fF1rIevAVgLLM8BAAAEIDQBAAAEIDSh3jwYdQEVsh6+jvXwNUjr4+tYD18DUHXsaQIAAAhApwkAACAAoQl1w8zuMbMfmdmgmX086npXJzd5AAAClklEQVSKZWY7zOxxM3vBzJ43sweirqkcZpYws6fN7KtR11IKM+sxs4fN7EUzO2Rmt0ZdUynM7Ldz/56eM7O/MrOWqGsC4orQhLpgZglJn5L0Lkl7Jf2cme2NtqqizUr6qLvvlXSLpI/U4New0AOSDkVdRBn+VNLX3f1qSdepBr8WM9sm6Tcl7XP3N0lKSPpAtFUB8UVoQr24WdKgux919xlJX5B0X8Q1FcXdX3f3p3JvTyj7Ir0t2qpKY2bbJb1H0kNR11IKM+uW9FZJn5Ykd59x99FoqypZUlKrmSUltUk6GXE9QGwRmlAvtkk6tuD946rRwCFJZrZL0g2Snoi2kpJ9UtLHJM1FXUiJdksakfTZ3BLjQ2bWHnVRxXL3E5L+SNJrkl6XNObu34i2KiC+CE1AjTGzDklfkvRb7j4edT3FMrN7JQ27+5NR11KGpKQbJf25u98g6bykWtwnt0HZjutuSVsltZvZB6OtCogvQhPqxQlJOxa8vz33WE0xs0ZlA9Pn3f2RqOsp0W2S3mtmryi7THqHmX0u2pKKdlzScXfPd/oeVjZE1Zp3SHrZ3UfcPS3pEUlvjrgmILYITagXP5B0pZntNrMmZTe7fiXimopiZqbsHppD7v7HUddTKnf/hLtvd/ddyv7/8C13r6nuhrufknTMzK7KPXSnpBciLKlUr0m6xczacv++7lQNbmgH1koy6gKAteDus2b265IeVfYOoc+4+/MRl1Ws2yT9oqRnzexg7rHfcfevRVhTPfsNSZ/PhfCjkj4ccT1Fc/cnzOxhSU8pe3fm02I6OLAsJoIDAAAEYHkOAAAgAKEJAAAgAKEJAAAgAKEJAAAgAKEJAAAgAKEJAAAgAKEJAAAgAKEJAAAgwP8HMMTsj0EpbcAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(2,1)\n",
    "fig.set_size_inches(10,10)\n",
    "ax[0].plot(np.array(losses).mean(axis=0))\n",
    "ax[1].imshow(grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import magent\n",
    "import math\n",
    "from scipy.spatial.distance import cityblock #A\n",
    "map_size = 30\n",
    "env = magent.GridWorld(\"battle\", map_size=map_size) #B\n",
    "env.set_render_dir(\"MAgent/build/render\") #C\n",
    "team1, team2 = env.get_handles() #D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "hid_layer = 25\n",
    "in_size = 359\n",
    "act_space = 21\n",
    "layers = [(in_size,hid_layer),(hid_layer,act_space)]\n",
    "params = gen_params(2,in_size*hid_layer+hid_layer*act_space) #A\n",
    "map_size = 30\n",
    "width = height = map_size\n",
    "n1 = n2 = 16 #B\n",
    "gap = 1 #C\n",
    "epochs = 100\n",
    "replay_size = 70\n",
    "batch_size = 25\n",
    "\n",
    "\n",
    "side1 = int(math.sqrt(n1)) * 2\n",
    "pos1 = []\n",
    "for x in range(width//2 - gap - side1, width//2 - gap - side1 + side1, 2): #D\n",
    "    for y in range((height - side1)//2, (height - side1)//2 + side1, 2):\n",
    "        pos1.append([x, y, 0])\n",
    "\n",
    "side2 = int(math.sqrt(n2)) * 2\n",
    "pos2 = []\n",
    "for x in range(width//2 + gap, width//2 + gap + side2, 2): #E\n",
    "    for y in range((height - side2)//2, (height - side2)//2 + side2, 2):\n",
    "        pos2.append([x, y, 0])\n",
    "        \n",
    "env.reset()\n",
    "env.add_agents(team1, method=\"custom\", pos=pos1) #F\n",
    "env.add_agents(team2, method=\"custom\", pos=pos2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x122bc5a58>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACzVJREFUeJzt3V/I3YV9x/H3Zy5G6h9I5hayVGYn3pTC4nhIB5XRUdo5Gag3Ui9KCrL0ooJCLybuol7KqJZdCXGGZsNZCip6IWtdEKQ3YpRMo9mqk0jNYtKSC+1gGvW7i+cXeJolz3PynD+/J37fLzg85/zO7zy/rz9855zf+fOcVBWS+vmdsQeQNA7jl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqmp353mxkluAv4BuAT4x6p6YLX1L83muozLp9mkpFX8L//DR/VhJlk36317b5JLgF8AXwfeBV4C7qiqN853m6uytb6cr61re5LW9mId4P06NVH80zzs3wW8VVVvV9VHwI+BW6b4fZIWaJr4dwC/XHH53WGZpIvAVMf8k0iyB9gDcBmfm/fmJE1omnv+Y8A1Ky5/flj2W6pqb1UtVdXSJjZPsTlJszRN/C8B1yf5QpJLgW8Cz8xmLEnztu6H/VX1cZK7gJ+y/FLfvqp6fWaTSZqrqY75q+pZ4NkZzSJpgXyHn9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTU31RZ1JjgIfAJ8AH1fV0iyGkjR/U8U/+Iuq+vUMfo+kBfJhv9TUtPEX8LMkLyfZc64VkuxJcjDJwdN8OOXmJM3KtA/7b6yqY0n+AHguyX9U1QsrV6iqvcBegKuytabcnqQZmeqev6qODT9PAk8Bu2YxlKT5W3f8SS5PcuWZ88A3gMOzGkzSfE3zsH8b8FSSM7/nX6rqX2cylaS5W3f8VfU28CcznEXSAvlSn9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/U1JrxJ9mX5GSSwyuWbU3yXJI3h59b5jumpFmb5J7/R8BNZy27FzhQVdcDB4bLki4ia8ZfVS8Ap85afAuwfzi/H7h1xnNJmrP1fkX3tqo6Ppx/D9h2vhWT7AH2AFzG59a5OUmzNvUTflVVQK1y/d6qWqqqpU1snnZzkmZkvfGfSLIdYPh5cnYjSVqE9cb/DLB7OL8beHo240halDWP+ZM8DnwVuDrJu8D3gQeAnyS5E3gHuH2eQ47lp/996LzX/eUf7pz57cba5nqtts21tjuP/855bfOzas34q+qO81z1tRnPImmBfIef1JTxS00Zv9SU8UtNGb/UVJbfoLcYV2VrfTm+SCDNy4t1gPfrVCZZ13t+qSnjl5oyfqkp45eaMn6pKeOXmlrvX/JpwU/1rc5P9V3cvOeXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmvIjvdJniB/plbQm45eaMn6pKeOXmjJ+qSnjl5qa5Is69wF/DZysqi8Ny+4H/gb41bDafVX17LyGHIsf6V2dH+m9uE1yz/8j4KZzLP9hVe0cTp+58KXPujXjr6oXgFMLmEXSAk1zzH9XkleT7EuyZWYTSVqI9cb/MHAdsBM4Djx4vhWT7ElyMMnB03y4zs1JmrV1xV9VJ6rqk6r6FHgE2LXKunuraqmqljaxeb1zSpqxdcWfZPuKi7cBh2czjqRFWfNTfUkeB74KXA2cAL4/XN4JFHAU+E5VHV9rY36qT5qvC/lU35qv81fVHedY/OgFTyVpQ/EdflJTxi81ZfxSU8YvNWX8UlPGLzXlt/Suwo/0rs6P9F7cvOeXmjJ+qSnjl5oyfqkp45eaMn6pKb+oU/oM8Ys6Ja3J+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2pqzfiTXJPk+SRvJHk9yd3D8q1Jnkvy5vBzy/zHlTQrk9zzfwx8r6q+CPwZ8N0kXwTuBQ5U1fXAgeGypIvEmvFX1fGqemU4/wFwBNgB3ALsH1bbD9w6ryElzd4FHfMnuRa4AXgR2FZVx4er3gO2zXQySXM1cfxJrgCeAO6pqvdXXlfLfw7onH8SKMmeJAeTHDzNh1MNK2l2Joo/ySaWw3+sqp4cFp9Isn24fjtw8ly3raq9VbVUVUub2DyLmSXNwCTP9gd4FDhSVQ+tuOoZYPdwfjfw9OzHkzQvk3xX31eAbwGvJTnzZWf3AQ8AP0lyJ/AOcPt8RpQ0D2vGX1U/B87310D9U7zSRcp3+ElNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1NQkX9F9TZLnk7yR5PUkdw/L709yLMmh4XTz/MeVNCuTfEX3x8D3quqVJFcCLyd5brjuh1X1g/mNJ2leJvmK7uPA8eH8B0mOADvmPZik+bqgY/4k1wI3AC8Oi+5K8mqSfUm2nOc2e5IcTHLwNB9ONayk2Zk4/iRXAE8A91TV+8DDwHXATpYfGTx4rttV1d6qWqqqpU1snsHIkmZhoviTbGI5/Meq6kmAqjpRVZ9U1afAI8Cu+Y0padYmebY/wKPAkap6aMXy7StWuw04PPvxJM3LJM/2fwX4FvBakkPDsvuAO5LsBAo4CnxnLhNKmotJnu3/OZBzXPXs7MeRtCi+w09qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfaipVtbiNJb8C3lmx6Grg1wsbYG3Os7qNNg9svJnGnuePqur3J1lxofH/v40nB6tqabQBzuI8q9to88DGm2mjzbMaH/ZLTRm/1NTY8e8deftnc57VbbR5YOPNtNHmOa9Rj/kljWfse35JIxkl/iQ3JfnPJG8luXeMGc6a52iS15IcSnJwpBn2JTmZ5PCKZVuTPJfkzeHnlpHnuT/JsWE/HUpy8wLnuSbJ80neSPJ6kruH5aPso1XmGW0fXaiFP+xPcgnwC+DrwLvAS8AdVfXGQgf57ZmOAktVNdrrs0n+HPgN8E9V9aVh2d8Dp6rqgeEfyS1V9bcjznM/8Juq+sEiZjhrnu3A9qp6JcmVwMvArcC3GWEfrTLP7Yy0jy7UGPf8u4C3qurtqvoI+DFwywhzbChV9QJw6qzFtwD7h/P7Wf6fa8x5RlNVx6vqleH8B8ARYAcj7aNV5rlojBH/DuCXKy6/y/g7rYCfJXk5yZ6RZ1lpW1UdH86/B2wbc5jBXUleHQ4LFnYYslKSa4EbgBfZAPvorHlgA+yjSfiE37Ibq+pPgb8Cvjs85N1Qavn4bOyXZh4GrgN2AseBBxc9QJIrgCeAe6rq/ZXXjbGPzjHP6PtoUmPEfwy4ZsXlzw/LRlNVx4afJ4GnWD402QhODMeWZ44xT445TFWdqKpPqupT4BEWvJ+SbGI5tMeq6slh8Wj76FzzjL2PLsQY8b8EXJ/kC0kuBb4JPDPCHAAkuXx4woYklwPfAA6vfquFeQbYPZzfDTw94ixn4jrjNha4n5IEeBQ4UlUPrbhqlH10vnnG3EcXrKoWfgJuZvkZ//8C/m6MGVbM8sfAvw+n18eaB3ic5YeJp1l+HuRO4PeAA8CbwL8BW0ee55+B14BXWY5u+wLnuZHlh/SvAoeG081j7aNV5hltH13oyXf4SU35hJ/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTf0f/B/laYNmmRcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(env.get_global_minimap(30,30)[:,:,:].sum(axis=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_neighbors(j,pos_list,r=6): #A\n",
    "    neighbors = []\n",
    "    pos_j = pos_list[j]\n",
    "    for i,pos in enumerate(pos_list):\n",
    "        if i == j:\n",
    "            continue\n",
    "        dist = cityblock(pos,pos_j)\n",
    "        if dist < r:\n",
    "            neighbors.append(i)\n",
    "    return neighbors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 4, 6, 7, 8, 9, 10, 13]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_neighbors(5,env.get_pos(team1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_onehot(a,l=21): #A\n",
    "    x = torch.zeros(21)\n",
    "    x[a] = 1\n",
    "    return x\n",
    "\n",
    "def get_scalar(v): #B\n",
    "    return torch.argmax(v)\n",
    "\n",
    "def get_mean_field(j,pos_list,act_list,r=7,l=21): #C\n",
    "    neighbors = get_neighbors(j,pos_list,r=r) #D\n",
    "    mean_field = torch.zeros(l)\n",
    "    for k in neighbors:\n",
    "        act_ = act_list[k]\n",
    "        act = get_onehot(act_)\n",
    "        mean_field += act\n",
    "    tot = mean_field.sum()\n",
    "    mean_field = mean_field / tot if tot > 0 else mean_field #E\n",
    "    return mean_field"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_acts(obs,param,layers,pos_list,acts,act_space=21,num_iter=5,temp=0.5):\n",
    "    N = acts.shape[0] #A\n",
    "    mean_fields = torch.zeros(N,act_space)\n",
    "    acts_ = acts.clone() #B\n",
    "    qvals = torch.zeros(N,act_space)\n",
    "\n",
    "    for i in range(num_iter): #C\n",
    "        for j in range(N): #D\n",
    "            mean_fields[j] = get_mean_field(j,pos_list,acts_)\n",
    "\n",
    "        for j in range(N): #E\n",
    "            state = torch.cat((obs[j].flatten(),mean_fields[j]))\n",
    "            qs = qfunc(state.detach(),param,layers=layers)\n",
    "            qvals[j,:] = qs[:]\n",
    "            acts_[j] = softmax_policy(qs.detach(),temp=temp)\n",
    "    return acts_, mean_fields, qvals\n",
    "\n",
    "def init_mean_field(N,act_space=21):\n",
    "    mean_fields = torch.abs(torch.rand(N,act_space))\n",
    "    for i in range(mean_fields.shape[0]):\n",
    "        mean_fields[i] = mean_fields[i] / mean_fields[i].sum()\n",
    "    return mean_fields"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(batch_size,replay,param,layers,J=64,gamma=0.5,lr=0.001):\n",
    "    ids = np.random.randint(low=0,high=len(replay),size=batch_size)\n",
    "    exps = [replay[idx] for idx in ids]\n",
    "    losses = []\n",
    "    jobs = torch.stack([ex[0] for ex in exps]).detach() #stack\n",
    "    jacts = torch.stack([ex[1] for ex in exps]).detach()\n",
    "    jrewards = torch.stack([ex[2] for ex in exps]).detach()\n",
    "    jmeans = torch.stack([ex[3] for ex in exps]).detach()\n",
    "    vs = torch.stack([ex[4] for ex in exps]).detach()\n",
    "    qs = []\n",
    "    for h in range(batch_size):\n",
    "        state = torch.cat((jobs[h].flatten(),jmeans[h]))\n",
    "        qs.append(qfunc(state.detach(),param,layers=layers))\n",
    "    qvals = torch.stack(qs)\n",
    "    target = qvals.clone().detach()\n",
    "    target[:,jacts] = jrewards + gamma * torch.max(vs,dim=1)[0] #20 = 20 + 20\n",
    "    loss = torch.sum(torch.pow(qvals - target.detach(),2))\n",
    "    losses.append(loss.detach().item())\n",
    "    loss.backward()\n",
    "    #SGD\n",
    "    with torch.no_grad():\n",
    "        param = param - lr * param.grad\n",
    "    param.requires_grad = True\n",
    "    return np.array(losses).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "N1 = env.get_num(team1) #A\n",
    "N2 = env.get_num(team2)\n",
    "step_ct = 0\n",
    "acts_1 = torch.randint(low=0,high=act_space,size=(N1,)) #B\n",
    "acts_2 = torch.randint(low=0,high=act_space,size=(N2,))\n",
    "\n",
    "replay1 = deque(maxlen=replay_size) #C\n",
    "replay2 = deque(maxlen=replay_size)\n",
    "\n",
    "qnext1 = torch.zeros(N1) #D\n",
    "qnext2 = torch.zeros(N2)\n",
    "\n",
    "act_means1 = init_mean_field(N1,act_space)  #E\n",
    "act_means2 = init_mean_field(N2,act_space)\n",
    "\n",
    "rewards1 = torch.zeros(N1) #F\n",
    "rewards2 = torch.zeros(N2)\n",
    "\n",
    "losses1 = []\n",
    "losses2 = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def team_step(team,param,acts,layers):\n",
    "    obs = env.get_observation(team) #A\n",
    "    ids = env.get_agent_id(team) #B\n",
    "    obs_small = torch.from_numpy(obs[0][:,:,:,[1,4]]) #C\n",
    "    agent_pos = env.get_pos(team) #D \n",
    "    acts, act_means, qvals = infer_acts(obs_small,\\\n",
    "                                       param,layers,agent_pos,acts) #E\n",
    "    return acts, act_means, qvals, obs_small, ids\n",
    "\n",
    "def add_to_replay(replay,obs_small, acts,rewards,act_means,qnext): #F\n",
    "    for j in range(rewards.shape[0]): #G\n",
    "        exp = (obs_small[j], acts[j],rewards[j],act_means[j],qnext[j])\n",
    "        replay.append(exp)\n",
    "        \n",
    "    return replay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 9.21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(epochs):\n",
    "    done = False\n",
    "    while not done: #A\n",
    "        acts_1, act_means1, qvals1, obs_small_1, ids_1 = \\\n",
    "            team_step(team1,params[0],acts_1,layers) #B\n",
    "        env.set_action(team1, acts_1.detach().numpy().astype(np.int32)) #C\n",
    "\n",
    "        acts_2, act_means2, qvals2, obs_small_2, ids_2 = \\\n",
    "            team_step(team2,params[0],acts_2,layers)\n",
    "        env.set_action(team2, acts_2.detach().numpy().astype(np.int32))\n",
    "\n",
    "        done = env.step() #D\n",
    "\n",
    "        _, _, qnext1, _, ids_1 = team_step(team1,params[0],acts_1,layers) #E\n",
    "        _, _, qnext2, _, ids_2 = team_step(team2,params[0],acts_2,layers)\n",
    "\n",
    "        env.render() #F\n",
    "\n",
    "        rewards1 = torch.from_numpy(env.get_reward(team1)).float() #G\n",
    "        rewards2 = torch.from_numpy(env.get_reward(team2)).float()\n",
    "        #\n",
    "        #\n",
    "        #\n",
    "        replay1 = add_to_replay(replay1, obs_small_1, acts_1,rewards1,act_means1,qnext1) #A\n",
    "        replay2 = add_to_replay(replay2, obs_small_2, acts_2,rewards2,act_means2,qnext2)     \n",
    "        shuffle(replay1) #B\n",
    "        shuffle(replay2)\n",
    "        \n",
    "        ids_1_ = list(zip(np.arange(ids_1.shape[0]),ids_1)) #C\n",
    "        ids_2_ = list(zip(np.arange(ids_2.shape[0]),ids_2))\n",
    "        \n",
    "        env.clear_dead() #D\n",
    "        \n",
    "        ids_1  = env.get_agent_id(team1) #E\n",
    "        ids_2  = env.get_agent_id(team2)\n",
    "        \n",
    "        ids_1_ = [i for (i,j) in ids_1_ if j in ids_1] #F\n",
    "        ids_2_ = [i for (i,j) in ids_2_ if j in ids_2]\n",
    "        \n",
    "        acts_1 = acts_1[ids_1_] #G\n",
    "        acts_2 = acts_2[ids_2_]\n",
    "        \n",
    "        step_ct += 1\n",
    "        if step_ct > 250:\n",
    "            break\n",
    "            \n",
    "        if len(replay1) > batch_size and len(replay2) > batch_size: #H\n",
    "            loss1 = train(batch_size,replay1,params[0],layers=layers,J=N1)\n",
    "            loss2 = train(batch_size,replay2,params[1],layers=layers,J=N1)\n",
    "            losses1.append(loss1)\n",
    "            losses2.append(loss2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
